From 3dc8a90982ecf3c5e2e2d9aceb331232ba7e4c4b Mon Sep 17 00:00:00 2001 From: Tao Wei Date: Mon, 23 Apr 2018 17:24:13 +0800 Subject: [PATCH 0001/1085] change to reference variable --- .../examples/android/jni/object_tracking/keypoint_detector.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/android/jni/object_tracking/keypoint_detector.cc b/tensorflow/examples/android/jni/object_tracking/keypoint_detector.cc index eb431328a7..567f1098b9 100644 --- a/tensorflow/examples/android/jni/object_tracking/keypoint_detector.cc +++ b/tensorflow/examples/android/jni/object_tracking/keypoint_detector.cc @@ -311,7 +311,7 @@ int KeypointDetector::AddExtraCandidatesForBoxes( return num_keypoints_added; } - Keypoint curr_keypoint = keypoints[num_keypoints_added++]; + Keypoint &curr_keypoint = keypoints[num_keypoints_added++]; curr_keypoint.pos_ = Point2f( box.left_ + box.GetWidth() * (i + 0.5f) / kNumToAddAsCandidates, box.top_ + box.GetHeight() * (j + 0.5f) / kNumToAddAsCandidates); -- GitLab From 071e6175dcc130b4c623e849a380d6434289eb66 Mon Sep 17 00:00:00 2001 From: Erik Smistad Date: Thu, 24 May 2018 15:47:00 +0200 Subject: [PATCH 0002/1085] Added the -Thost=x64 flag to cmake build instructions --- tensorflow/contrib/cmake/README.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md index 0b79f718d4..5c203b777c 100644 --- a/tensorflow/contrib/cmake/README.md +++ b/tensorflow/contrib/cmake/README.md @@ -106,17 +106,6 @@ Step-by-step Windows build 1. Install the prerequisites detailed above, and set up your environment. - * The following commands assume that you are using the Windows Command - Prompt (`cmd.exe`). You will need to set up your environment to use the - appropriate toolchain, i.e. the 64-bit tools. (Some of the binary targets - we will build are too large for the 32-bit tools, and they will fail with - out-of-memory errors.) The typical command to do set up your - environment is: - - ``` - D:\temp> "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvarsall.bat" - ``` - * When building with GPU support after installing the CUDNN zip file from NVidia, append its bin directory to your PATH environment variable. In case TensorFlow fails to find the CUDA dll's during initialization, check your PATH environment variable. @@ -168,7 +157,7 @@ Step-by-step Windows build and must be the last character on each line. ``` - D:\...\build> cmake .. -A x64 -DCMAKE_BUILD_TYPE=Release ^ + D:\...\build> cmake .. -A x64 -Thost=x64 -DCMAKE_BUILD_TYPE=Release ^ More? -DSWIG_EXECUTABLE=C:/tools/swigwin-3.0.10/swig.exe ^ More? -DPYTHON_EXECUTABLE=C:/Users/%USERNAME%/AppData/Local/Continuum/Anaconda3/python.exe ^ More? -DPYTHON_LIBRARIES=C:/Users/%USERNAME%/AppData/Local/Continuum/Anaconda3/libs/python35.lib @@ -197,6 +186,10 @@ Step-by-step Windows build not currently supported, because it relies on a `Debug` library for Python (`python35d.lib`) that is not distributed by default. + The `-Thost=x64` flag will ensure that the 64 bit compiler and linker + is used when building. Without this flag, MSBuild will use the 32 bit + toolchain which is prone to compile errors such as "compiler out of heap space". + There are various options that can be specified when generating the solution and project files: @@ -263,6 +256,11 @@ Step-by-step Windows build 4. Invoke MSBuild to build TensorFlow. + Set up the path to find MSbuild: + ``` + D:\temp> "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvarsall.bat" + ``` + To build the C++ example program, which will be created as a `.exe` executable in the subdirectory `.\Release`: -- GitLab From 6890731b2693f6b71dedaca6b2eaf8b488226836 Mon Sep 17 00:00:00 2001 From: Erik Smistad Date: Thu, 24 May 2018 15:47:22 +0200 Subject: [PATCH 0003/1085] increase minimum cmake version required to 3.8 --- tensorflow/contrib/cmake/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 0708d6b7b9..225c5e6227 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -1,5 +1,9 @@ # Minimum CMake required -cmake_minimum_required(VERSION 3.5) +if(WIN32) + cmake_minimum_required(VERSION 3.8) +else() + cmake_minimum_required(VERSION 3.5) +endif() # Project project(tensorflow C CXX) -- GitLab From ff94f56d0fd8b4b3043c79cc7ad1f939b587e44c Mon Sep 17 00:00:00 2001 From: Yuxin Wu Date: Wed, 13 Jun 2018 06:41:22 -0700 Subject: [PATCH 0004/1085] Update non_max_suppression_op.cc --- tensorflow/core/kernels/non_max_suppression_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc index 23fdfe944a..4537ccb14b 100644 --- a/tensorflow/core/kernels/non_max_suppression_op.cc +++ b/tensorflow/core/kernels/non_max_suppression_op.cc @@ -131,7 +131,7 @@ void DoNonMaxSuppressionOp(OpKernelContext* context, const Tensor& boxes, // therefore we iterate through the previously selected boxes backwards // in order to see if `next_candidate` should be suppressed. bool should_select = true; - for (int j = selected.size() - 1; j >= 0; --j) { + for (int j = static_cast(selected.size()) - 1; j >= 0; --j) { iou = IOU(boxes_data, next_candidate.box_index, selected[j]); if (iou == 0.0) continue; if (iou > iou_threshold) should_select = false; -- GitLab From 667b9995267ff3e2592dca26d7a5748437ebb820 Mon Sep 17 00:00:00 2001 From: maxpumperla Date: Thu, 14 Jun 2018 12:14:29 +0200 Subject: [PATCH 0005/1085] More documentation for keras adagrad and adadelta --- tensorflow/python/keras/optimizers.py | 60 +++++++++++++++++---------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py index f58aeaea1a..629972fd8e 100644 --- a/tensorflow/python/keras/optimizers.py +++ b/tensorflow/python/keras/optimizers.py @@ -314,17 +314,24 @@ class RMSprop(Optimizer): @tf_export('keras.optimizers.Adagrad') class Adagrad(Optimizer): - """Adagrad optimizer. + """Adagrad optimizer. - It is recommended to leave the parameters of this optimizer - at their default values. + Adagrad is an optimizer with parameter-specific learning rates, + which are adapted relative to how frequently a parameter gets + updated during training. The more updates a parameter receives, + the smaller the updates. - Arguments: - lr: float >= 0. Learning rate. - epsilon: float >= 0. If `None`, defaults to `K.epsilon()`. - decay: float >= 0. Learning rate decay over each update. + It is recommended to leave the parameters of this optimizer + at their default values. - """ + # Arguments + lr: float >= 0. Initial learning rate. + epsilon: float >= 0. If `None`, defaults to `K.epsilon()`. + decay: float >= 0. Learning rate decay over each update. + + # References + - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + """ def __init__(self, lr=0.01, epsilon=None, decay=0., **kwargs): super(Adagrad, self).__init__(**kwargs) @@ -374,19 +381,30 @@ class Adagrad(Optimizer): @tf_export('keras.optimizers.Adadelta') class Adadelta(Optimizer): - """Adadelta optimizer. - - It is recommended to leave the parameters of this optimizer - at their default values. - - Arguments: - lr: float >= 0. Learning rate. - It is recommended to leave it at the default value. - rho: float >= 0. - epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. - decay: float >= 0. Learning rate decay over each update. - - """ + """Adadelta optimizer. + + Adadelta is a more robust extension of Adagrad + that adapts learning rates based on a moving window of gradient updates, + instead of accumulating all past gradients. This way, Adadelta continues + learning even when many updates have been done. Compared to Adagrad, in the + original version of Adadelta you don't have to set an initial learning + rate. In this version, initial learning rate and decay factor can + be set, as in most other Keras optimizers. + + It is recommended to leave the parameters of this optimizer + at their default values. + + # Arguments + lr: float >= 0. Initial learning rate, defaults to 1. + It is recommended to leave it at the default value. + rho: float >= 0. Adadelta decay factor, corresponding to fraction of + gradient to keep at each time step. + epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. + decay: float >= 0. Initial learning rate decay. + + # References + - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701) + """ def __init__(self, lr=1.0, rho=0.95, epsilon=None, decay=0., **kwargs): super(Adadelta, self).__init__(**kwargs) -- GitLab From 2e436951bb63a0294848b6f6d3746e449a305ad1 Mon Sep 17 00:00:00 2001 From: Stefan Dyulgerov Date: Tue, 17 Jul 2018 22:37:19 +0300 Subject: [PATCH 0006/1085] version_info.cc generated only once version_info.cc in the cmake files is generated every time when we build tensorflow and this forces rebuild of the whole project, since it is in the core library. added make.bat for windows, which does the same as make.sh to be executed easily from a build machine. the default now is visual studio 17 --- tensorflow/contrib/cmake/make.bat | 38 +++++++++++++++++++ .../contrib/cmake/tf_core_framework.cmake | 23 +++++++---- 2 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 tensorflow/contrib/cmake/make.bat diff --git a/tensorflow/contrib/cmake/make.bat b/tensorflow/contrib/cmake/make.bat new file mode 100644 index 0000000000..d52b24e01d --- /dev/null +++ b/tensorflow/contrib/cmake/make.bat @@ -0,0 +1,38 @@ +%echo off + +cd /d %~dp0 + +if exist _build rd /s /q _build + +mkdir _build +chdir _build + + +rem cmake ../ -G "Visual Studio 15 Win64" -DCMAKE_GENERATOR_TOOLSET=v141,host=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install + +CALL :NORMALIZEPATH "..\..\..\.." +SET SOURCE_DIR=%RETVAL% + +echo %SOURCE_DIR% + +SET SOURCE_DIR=F:\frameworks\tensorflow\ + +CALL :NORMALIZEPATH "../../../tools/git/gen_git_source.py" +SET SOURCE_PYTHON_SCRIPT=%RETVAL% + +CALL :NORMALIZEPATH "../../../core/util/version_info.cc" +SET SOURCE_VERSION_CC=%RETVAL% + +python %SOURCE_PYTHON_SCRIPT% --raw_generate %SOURCE_VERSION_CC% --source_dir %SOURCE_DIR% --git_tag_override= + +cmake ../ -G "Visual Studio 15 Win64" -DCMAKE_GENERATOR_TOOLSET=v141,host=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install + +EXIT /B + +:NORMALIZEPATH + SET RETVAL=%~dpfn1 + EXIT /B + + + + \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index 067c299a71..7e806685b8 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -258,14 +258,21 @@ add_dependencies(tf_core_lib ${tensorflow_EXTERNAL_DEPENDENCIES} tf_protos_cc) # force_rebuild always runs forcing ${VERSION_INFO_CC} target to run # ${VERSION_INFO_CC} would cache, but it depends on a phony never produced # target. -set(VERSION_INFO_CC ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) -add_custom_target(force_rebuild_target ALL DEPENDS ${VERSION_INFO_CC}) -add_custom_command(OUTPUT __force_rebuild COMMAND ${CMAKE_COMMAND} -E echo) -add_custom_command(OUTPUT - ${VERSION_INFO_CC} - COMMAND ${PYTHON_EXECUTABLE} ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py - ARGS --raw_generate ${VERSION_INFO_CC} --source_dir ${tensorflow_source_dir} --git_tag_override=${GIT_TAG_OVERRIDE} - DEPENDS __force_rebuild) +# This code forces rebuild every time, not needed as version from git is fetched only once +# move to make.bat which mimicks make.sh + +if (NOT WIN32) + + set(VERSION_INFO_CC ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) + add_custom_target(force_rebuild_target ALL DEPENDS ${VERSION_INFO_CC}) + add_custom_command(OUTPUT __force_rebuild COMMAND ${CMAKE_COMMAND} -E echo) + add_custom_command(OUTPUT + ${VERSION_INFO_CC} + COMMAND ${PYTHON_EXECUTABLE} ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py + ARGS --raw_generate ${VERSION_INFO_CC} --source_dir ${tensorflow_source_dir} --git_tag_override=${GIT_TAG_OVERRIDE} + DEPENDS __force_rebuild) +endif() + set(tf_version_srcs ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) ######################################################## -- GitLab From 097d891caff16fd5fe47f4655650e2bbb7aa659e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 30 Jul 2018 20:50:03 +0000 Subject: [PATCH 0007/1085] Enable int64 support for MatMul This fix tries to address the issue raised in 21241 where there were no int64 support for MatMul. This fix adds int64 support for MatMul. This fix fixes 21241. Signed-off-by: Yong Tang --- tensorflow/core/kernels/matmul_op.cc | 1 + tensorflow/core/ops/math_ops.cc | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/matmul_op.cc b/tensorflow/core/kernels/matmul_op.cc index 80376c61aa..4317054f91 100644 --- a/tensorflow/core/kernels/matmul_op.cc +++ b/tensorflow/core/kernels/matmul_op.cc @@ -598,6 +598,7 @@ TF_CALL_half(REGISTER_CPU); TF_CALL_bfloat16(REGISTER_CPU); TF_CALL_int32(REGISTER_CPU); +TF_CALL_int64(REGISTER_CPU); TF_CALL_complex64(REGISTER_CPU); TF_CALL_complex128(REGISTER_CPU); #endif diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 1667c398f4..a0e920fb80 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -743,7 +743,9 @@ REGISTER_OP("MatMul") .Output("product: T") .Attr("transpose_a: bool = false") .Attr("transpose_b: bool = false") - .Attr("T: {bfloat16, half, float, double, int32, complex64, complex128}") + .Attr( + "T: {bfloat16, half, float, double, int32, int64, complex64, " + "complex128}") .SetShapeFn(shape_inference::MatMulShape); REGISTER_OP("SparseMatMul") -- GitLab From 16a50fcbacb8e46f6c4560a6e58ed26f5fd2d133 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 30 Jul 2018 20:53:06 +0000 Subject: [PATCH 0008/1085] Enable int64 test cases for MatMul Signed-off-by: Yong Tang --- tensorflow/python/kernel_tests/matmul_op_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py index b167278984..36ffdc08ed 100644 --- a/tensorflow/python/kernel_tests/matmul_op_test.py +++ b/tensorflow/python/kernel_tests/matmul_op_test.py @@ -102,7 +102,7 @@ class MatMulGradientTest(test_lib.TestCase): def _GetMatMulGradientTest(a_np_, b_np_, use_static_shape_, **kwargs_): def Test(self): - if not use_static_shape_ or a_np_.dtype in (np.int32, np.float16): + if not use_static_shape_ or a_np_.dtype in (np.int32, np.int64, np.float16): self.skipTest("Skipping infeasible gradient test.") # Transpose and possibly conjugate a_np_ and b_np_ according to the @@ -214,9 +214,9 @@ if __name__ == "__main__": sizes = [1, 3, 5] trans_options = [[False, False], [True, False], [False, True]] for use_static_shape in [False, True]: - for dtype in (np.int32, np.float16, np.float32, np.float64, np.complex64, - np.complex128): - if not use_static_shape and dtype == np.int32: + for dtype in (np.int32, np.int64, np.float16, np.float32, np.float64, + np.complex64, np.complex128): + if not use_static_shape and dtype == np.int32 and dtype == np.int64: # TODO(rmlarsen): Re-enable this test when we have fixed the underlying # bug in Windows (b/35935459). continue -- GitLab From 29eaaf0ffd96002bac06b2e41fbcc246bc5023f6 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 30 Jul 2018 20:54:29 +0000 Subject: [PATCH 0009/1085] Also enables int64 support for BatchMatMul, as test cases uses BatchMatMul as well Signed-off-by: Yong Tang --- tensorflow/core/kernels/batch_matmul_op_real.cc | 1 + tensorflow/core/ops/math_ops.cc | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/batch_matmul_op_real.cc b/tensorflow/core/kernels/batch_matmul_op_real.cc index fe259c1634..3f560023a3 100644 --- a/tensorflow/core/kernels/batch_matmul_op_real.cc +++ b/tensorflow/core/kernels/batch_matmul_op_real.cc @@ -27,6 +27,7 @@ TF_CALL_double(REGISTER_BATCH_MATMUL_CPU); #endif TF_CALL_half(REGISTER_BATCH_MATMUL_CPU); TF_CALL_int32(REGISTER_BATCH_MATMUL_CPU); +TF_CALL_int64(REGISTER_BATCH_MATMUL_CPU); #if GOOGLE_CUDA TF_CALL_float(REGISTER_BATCH_MATMUL_GPU); diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index a0e920fb80..5151b75650 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -65,7 +65,9 @@ REGISTER_OP("BatchMatMul") .Input("x: T") .Input("y: T") .Output("output: T") - .Attr("T: {bfloat16, half, float, double, int32, complex64, complex128}") + .Attr( + "T: {bfloat16, half, float, double, int32, int64, complex64, " + "complex128}") .Attr("adj_x: bool = false") .Attr("adj_y: bool = false") .SetShapeFn([](InferenceContext* c) { -- GitLab From 6a0bffab81dc5d39a18fbc52d6c71509e016985d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Jul 2018 20:14:42 +0000 Subject: [PATCH 0010/1085] Fix bug in test case of matmul Signed-off-by: Yong Tang --- tensorflow/python/kernel_tests/matmul_op_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py index 36ffdc08ed..493b3cf9d4 100644 --- a/tensorflow/python/kernel_tests/matmul_op_test.py +++ b/tensorflow/python/kernel_tests/matmul_op_test.py @@ -216,7 +216,7 @@ if __name__ == "__main__": for use_static_shape in [False, True]: for dtype in (np.int32, np.int64, np.float16, np.float32, np.float64, np.complex64, np.complex128): - if not use_static_shape and dtype == np.int32 and dtype == np.int64: + if not use_static_shape and (dtype == np.int32 or dtype == np.int64): # TODO(rmlarsen): Re-enable this test when we have fixed the underlying # bug in Windows (b/35935459). continue -- GitLab From dc93bc7127548da89f0e6d7a0e22986f6a5780d9 Mon Sep 17 00:00:00 2001 From: silent567 Date: Tue, 7 Aug 2018 22:28:23 +0800 Subject: [PATCH 0011/1085] Change the docs of the convolution function The ranks of input and filter should be N+2 instead of N. Because the function computes N-D convolution, the input's rank should be N+2 with additional batch and input_channel dimensions, and the filter's rank should be N+2 with additional input_channel and output_channel dimensions. --- tensorflow/python/ops/nn_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 5cdb7726a7..f8cf044e9a 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -711,12 +711,12 @@ def convolution( It is required that 1 <= N <= 3. Args: - input: An N-D `Tensor` of type `T`, of shape + input: An (N+2)-D `Tensor` of type `T`, of shape `[batch_size] + input_spatial_shape + [in_channels]` if data_format does not start with "NC" (default), or `[batch_size, in_channels] + input_spatial_shape` if data_format starts with "NC". - filter: An N-D `Tensor` with the same type as `input` and shape + filter: An (N+2)-D `Tensor` with the same type as `input` and shape `spatial_filter_shape + [in_channels, out_channels]`. padding: A string, either `"VALID"` or `"SAME"`. The padding algorithm. strides: Optional. Sequence of N ints >= 1. Specifies the output stride. -- GitLab From 29f596cf21f0332c1e2ece8798fdd9fefd2ba947 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 4 Jun 2018 14:04:59 +0000 Subject: [PATCH 0012/1085] Improve the shape function of Bincount There was not a lot of restriction in shape function of Bincount and the output shape was unknown. It is actually possible to get a better shape output if `size` input is known. This fix adds enhancement to the shape function of Bincount. Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 1667c398f4..7d0f29368b 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1416,6 +1416,10 @@ REGISTER_OP("Bincount") .Attr("T: {int32, int64, float32, float64}") .Output("bins: T") .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; + // The input `size` must be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + c->set_output(0, c->UnknownShapeOfRank(1)); return Status::OK(); }); -- GitLab From 740c58b6fa5b6e1c85f688fbda322da0231aa169 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 4 Jun 2018 14:44:44 +0000 Subject: [PATCH 0013/1085] Return `[size]` shape if size is known for Bincount. Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 7d0f29368b..b57385f63b 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1420,7 +1420,19 @@ REGISTER_OP("Bincount") // The input `size` must be a scalar. TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - c->set_output(0, c->UnknownShapeOfRank(1)); + const Tensor* size_tensor = c->input_tensor(1); + if (size_tensor == nullptr) { + // Return unknown shape if size is not known. + c->set_output(0, c->UnknownShapeOfRank(1)); + return Status::OK(); + } + + // Return `[size]` shape if size is known. + int32 size_val = size_tensor->scalar()(); + if (size_val < 0) { + return errors::InvalidArgument("size (", size_val, ") must be non-negative"); + } + c->set_output(0, c->MakeShape({size_val})); return Status::OK(); }); -- GitLab From e6981fc2225a529427391e98f492eee7bb865988 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 11 Aug 2018 18:39:13 +0000 Subject: [PATCH 0014/1085] Add additional test cases for Bincount Shape function, and fix clang-format issue Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 3 ++- tensorflow/core/ops/math_ops_test.cc | 12 ++++++++++++ .../python/kernel_tests/bincount_op_test.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index b57385f63b..0ba4a9a005 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1430,7 +1430,8 @@ REGISTER_OP("Bincount") // Return `[size]` shape if size is known. int32 size_val = size_tensor->scalar()(); if (size_val < 0) { - return errors::InvalidArgument("size (", size_val, ") must be non-negative"); + return errors::InvalidArgument("size (", size_val, + ") must be non-negative"); } c->set_output(0, c->MakeShape({size_val})); return Status::OK(); diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index 23f1538912..7bf7c476f4 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -558,4 +558,16 @@ TEST(MathOpsTest, QuantizedAdd_ShapeFn) { INFER_ERROR("must be rank 0", op, "?;?;?;?;[3];?"); INFER_ERROR("must be rank 0", op, "?;?;?;?;?;[4]"); } + +TEST(MathOpsTest, Bincount_ShapeFn) { + ShapeInferenceTestOp op("Bincount"); + + // size should be scalar. + INFER_ERROR("Shape must be rank 0 but is rank 1", op, "?;[1];?"); + + INFER_OK(op, "?;?;?", "[?]"); + INFER_OK(op, "?;[];?", "[?]"); + INFER_OK(op, "[?];[];?", "[?]"); + INFER_OK(op, "[?];[];[?]", "[?]"); +} } // end namespace tensorflow diff --git a/tensorflow/python/kernel_tests/bincount_op_test.py b/tensorflow/python/kernel_tests/bincount_op_test.py index 2767df127e..15d9de56db 100644 --- a/tensorflow/python/kernel_tests/bincount_op_test.py +++ b/tensorflow/python/kernel_tests/bincount_op_test.py @@ -22,6 +22,8 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import googletest @@ -97,6 +99,23 @@ class BincountTest(test_util.TensorFlowTestCase): with self.assertRaises(errors.InvalidArgumentError): math_ops.bincount([1, 2, 3, -1, 6, 8]).eval() + def test_shape_function(self): + # size must be scalar. + with self.assertRaisesRegexp( + ValueError, "Shape must be rank 0 but is rank 1 for 'Bincount'"): + gen_math_ops.bincount([1, 2, 3, -1, 6, 8], [1], []) + # size must be positive. + with self.assertRaisesRegexp( + ValueError, "must be non-negative"): + gen_math_ops.bincount([1, 2, 3, -1, 6, 8], -5, []) + # if size is a constant then the shape is known. + v1 = gen_math_ops.bincount([1, 2, 3, -1, 6, 8], 5, []) + self.assertAllEqual(v1.get_shape().as_list(), [5]) + # if size is a placeholder then the shape is unknown. + s = array_ops.placeholder(dtype=dtypes.int32) + v2 = gen_math_ops.bincount([1, 2, 3, -1, 6, 8], s, []) + self.assertAllEqual(v2.get_shape().as_list(), [None]) + if __name__ == "__main__": googletest.main() -- GitLab From cea293f00cd8710e439937b048745c14ac52b3ff Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 15 May 2018 22:40:27 +0000 Subject: [PATCH 0015/1085] Fix tf.matching_files issue with access denied subdirectory. This fix tries to address the issue raised in 19274 where tf.matching_files will return an error if no permission for subdirectories (Even if the subdirectories does not match). This fix addresses the issue by ignore the case for `permisson denied` during the directory traversal. This fix fixes 19274. Signed-off-by: Yong Tang --- tensorflow/core/platform/file_system_helper.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/file_system_helper.cc b/tensorflow/core/platform/file_system_helper.cc index 0ba0e6304f..1e72e9e41e 100644 --- a/tensorflow/core/platform/file_system_helper.cc +++ b/tensorflow/core/platform/file_system_helper.cc @@ -82,7 +82,10 @@ Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern, dir_q.pop_front(); std::vector children; Status s = fs->GetChildren(current_dir, &children); - ret.Update(s); + // We will ignore permission denied error, and update status otherwise. + if (s.code() != tensorflow::error::PERMISSION_DENIED) { + ret.Update(s); + } if (children.empty()) continue; // This IsDirectory call can be expensive for some FS. Parallelizing it. children_dir_status.resize(children.size()); -- GitLab From be275e00c9134ef3e7cce1b0806f62aec28b6945 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 11 Aug 2018 21:53:02 +0000 Subject: [PATCH 0016/1085] Update matching_files and bail in case PERMISSION_DENIED is encountered, update based on review comment. Signed-off-by: Yong Tang --- tensorflow/core/platform/file_system_helper.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/platform/file_system_helper.cc b/tensorflow/core/platform/file_system_helper.cc index 1e72e9e41e..a26a224e14 100644 --- a/tensorflow/core/platform/file_system_helper.cc +++ b/tensorflow/core/platform/file_system_helper.cc @@ -82,10 +82,11 @@ Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern, dir_q.pop_front(); std::vector children; Status s = fs->GetChildren(current_dir, &children); - // We will ignore permission denied error, and update status otherwise. - if (s.code() != tensorflow::error::PERMISSION_DENIED) { - ret.Update(s); + // In case PERMISSION_DENIED is encountered, we bail here. + if (s.code() == tensorflow::error::PERMISSION_DENIED) { + continue; } + ret.Update(s); if (children.empty()) continue; // This IsDirectory call can be expensive for some FS. Parallelizing it. children_dir_status.resize(children.size()); -- GitLab From c1731614e10e5b2a8e77f1ee4565b3185541483c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 11 Aug 2018 21:53:56 +0000 Subject: [PATCH 0017/1085] Add test case for matching_files when PERMISSION_DENIED is encountered in subdir Signed-off-by: Yong Tang --- tensorflow/python/lib/io/file_io_test.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py index c21eb93103..1d247aa8ba 100644 --- a/tensorflow/python/lib/io/file_io_test.py +++ b/tensorflow/python/lib/io/file_io_test.py @@ -23,7 +23,9 @@ import os.path from tensorflow.python.framework import errors from tensorflow.python.lib.io import file_io +from tensorflow.python.ops import gen_io_ops from tensorflow.python.platform import test +from tensorflow.python.util import compat class FileIoTest(test.TestCase): @@ -582,5 +584,31 @@ class FileIoTest(test.TestCase): self.assertTrue(crc1 != crc2) self.assertEqual(crc2, crc3) + def testMatchingFilesPermission(self): + # Test case for GitHub issue 19274. + # Create top level directory test_dir. + dir_path = os.path.join(self._base_dir, "test_dir") + file_io.create_dir(dir_path) + # Create second level directories `noread` and `any`. + noread_path = os.path.join(dir_path, "noread") + file_io.create_dir(noread_path) + any_path = os.path.join(dir_path, "any") + file_io.create_dir(any_path) + files = ["file1.txt", "file2.txt", "file3.txt"] + for name in files: + file_path = os.path.join(any_path, name) + file_io.FileIO(file_path, mode="w").write("testing") + file_path = os.path.join(noread_path, "file4.txt") + file_io.FileIO(file_path, mode="w").write("testing") + # Change noread to noread access. + os.chmod(noread_path, 0) + expected_match = [compat.as_bytes(dir_path)] + with self.test_session() as sess: + self.assertItemsEqual( + gen_io_ops.matching_files(dir_path).eval(), expected_match) + # Change noread back so that it could be cleaned during tearDown. + os.chmod(noread_path, 0o777) + + if __name__ == "__main__": test.main() -- GitLab From 537bd0d8237d77c789c1b7633d8ba4b68007f52e Mon Sep 17 00:00:00 2001 From: Andy Craze Date: Sun, 12 Aug 2018 14:40:04 -0700 Subject: [PATCH 0018/1085] Update Nesterov implementation docs Clarification that this is a modified version of the algorithm which is only correct under certain conditions Fixes #19899 --- tensorflow/python/training/momentum.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py index cb3ec6f053..34c74cda4e 100644 --- a/tensorflow/python/training/momentum.py +++ b/tensorflow/python/training/momentum.py @@ -59,6 +59,10 @@ class MomentumOptimizer(optimizer.Optimizer): This implementation always computes gradients at the value of the variable(s) passed to the optimizer. Using Nesterov Momentum makes the variable(s) track the values called `theta_t + mu*v_t` in the paper. + This implementation is an approximation of the original formula, valid + for high values of momentum. It will compute the "adjusted gradient" in NAG + by assuming that the new gradient will be estimated by the current + average gradient plus the product of momentum and the change in the average gradient. @compatibility(eager) When eager execution is enabled, `learning_rate` and `momentum` can each be -- GitLab From 9c21d22957a2d1e1f52464b923bf1bf5cfc29d5e Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Mon, 13 Aug 2018 18:02:23 -0700 Subject: [PATCH 0019/1085] Ran clang_format tool --- tensorflow/core/graph/mkl_layout_pass.cc | 236 +++++++++++++++++- .../core/graph/mkl_tfconversion_pass.cc | 18 +- tensorflow/core/kernels/mkl_tfconv_op.h | 1 + 3 files changed, 241 insertions(+), 14 deletions(-) diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc index c22e0a3872..c0ff6e10c1 100644 --- a/tensorflow/core/graph/mkl_layout_pass.cc +++ b/tensorflow/core/graph/mkl_layout_pass.cc @@ -2432,8 +2432,37 @@ class MklLayoutRewritePass : public GraphOptimizationPass { csinfo_.mkl_conv2d_with_bias = "_MklConv2DWithBias"; csinfo_.mkl_conv2d_grad_filter_with_bias = "_MklConv2DBackpropFilterWithBias"; +// Temporarily don't convert quantized operators into MKL versions for now. +// TODO(Intel-tf) Once all the relevant PRs have been merged then remove +// the ifdef. +#ifdef INTEL_MKL_QUANTIZED + csinfo_.quantized_avg_pool = "QuantizedAvgPool"; + csinfo_.quantized_concatv2 = "QuantizedConcatV2"; + csinfo_.quantized_conv2d = "QuantizedConv2D"; + csinfo_.quantized_conv2d_with_requantize = "QuantizedConv2DAndRequantize"; + csinfo_.quantized_conv2d_with_bias = "QuantizedConv2DWithBias"; + csinfo_.quantized_conv2d_with_bias_and_requantize = + "QuantizedConv2DWithBiasAndRequantize"; + csinfo_.quantized_conv2d_and_relu = "QuantizedConv2DAndRelu"; + csinfo_.quantized_conv2d_and_relu_and_requantize = + "QuantizedConv2DAndReluAndRequantize"; + csinfo_.quantized_conv2d_with_bias_and_relu = + "QuantizedConv2DWithBiasAndRelu"; + csinfo_.quantized_conv2d_with_bias_and_relu_and_requantize = + "QuantizedConv2DWithBiasAndReluAndRequantize"; + csinfo_.quantized_max_pool = "QuantizedMaxPool"; + csinfo_.quantized_conv2d_with_bias_sum_and_relu = + "QuantizedConv2DWithBiasSumAndRelu"; + csinfo_.quantized_conv2d_with_bias_sum_and_relu_and_requantize = + "QuantizedConv2DWithBiasSumAndReluAndRequantize"; + csinfo_.quant_conv2d_with_bias_signed_sum_and_relu_and_requantize = + "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"; +#endif csinfo_.relu = "Relu"; csinfo_.relu_grad = "ReluGrad"; +#ifdef INTEL_MKL_QUANTIZED + csinfo_.requantize = "Requantize"; +#endif csinfo_.tanh = "Tanh"; csinfo_.tanh_grad = "TanhGrad"; csinfo_.reshape = "Reshape"; @@ -2508,11 +2537,73 @@ class MklLayoutRewritePass : public GraphOptimizationPass { rinfo_.push_back({csinfo_.mul, mkl_op_registry::GetMklOpName(csinfo_.mul), CopyAttrsDataType, AlwaysRewrite}); +#ifdef INTEL_MKL_QUANTIZED + rinfo_.push_back({csinfo_.quantized_avg_pool, + mkl_op_registry::GetMklOpName(csinfo_.quantized_avg_pool), + CopyAttrsQuantizedPooling, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_concatv2, + mkl_op_registry::GetMklOpName(csinfo_.quantized_concatv2), + CopyAttrsConcatV2, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d, + mkl_op_registry::GetMklOpName(csinfo_.quantized_conv2d), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d_with_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back( + {csinfo_.quantized_conv2d_with_bias, + mkl_op_registry::GetMklOpName(csinfo_.quantized_conv2d_with_bias), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d_with_bias_and_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_bias_and_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back( + {csinfo_.quantized_conv2d_and_relu, + mkl_op_registry::GetMklOpName(csinfo_.quantized_conv2d_and_relu), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d_and_relu_and_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_and_relu_and_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d_with_bias_and_relu, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_bias_and_relu), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back( + {csinfo_.quantized_conv2d_with_bias_and_relu_and_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_bias_and_relu_and_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_max_pool, + mkl_op_registry::GetMklOpName(csinfo_.quantized_max_pool), + CopyAttrsQuantizedPooling, AlwaysRewrite}); + rinfo_.push_back({csinfo_.quantized_conv2d_with_bias_sum_and_relu, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_bias_sum_and_relu), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back( + {csinfo_.quantized_conv2d_with_bias_sum_and_relu_and_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quantized_conv2d_with_bias_sum_and_relu_and_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); + rinfo_.push_back( + {csinfo_.quant_conv2d_with_bias_signed_sum_and_relu_and_requantize, + mkl_op_registry::GetMklOpName( + csinfo_.quant_conv2d_with_bias_signed_sum_and_relu_and_requantize), + CopyAttrsQuantizedConv2D, AlwaysRewrite}); +#endif rinfo_.push_back({csinfo_.relu, mkl_op_registry::GetMklOpName(csinfo_.relu), CopyAttrsDataType, AlwaysRewrite}); rinfo_.push_back({csinfo_.relu_grad, mkl_op_registry::GetMklOpName(csinfo_.relu_grad), CopyAttrsDataType, AlwaysRewrite}); +#ifdef INTEL_MKL_QUANTIZED + rinfo_.push_back({csinfo_.requantize, + mkl_op_registry::GetMklOpName(csinfo_.requantize), + CopyAttrsRequantize, AlwaysRewrite}); +#endif /* rinfo_.push_back({csinfo_.tanh, mkl_op_registry::GetMklOpName(csinfo_.tanh), @@ -2629,8 +2720,23 @@ class MklLayoutRewritePass : public GraphOptimizationPass { string mkl_conv2d_grad_filter_with_bias; string mkl_conv2d_with_bias; string mul; + string quantized_avg_pool; + string quantized_conv2d; + string quantized_conv2d_with_requantize; + string quantized_conv2d_with_bias; + string quantized_conv2d_with_bias_and_requantize; + string quantized_conv2d_and_relu; + string quantized_conv2d_and_relu_and_requantize; + string quantized_conv2d_with_bias_and_relu; + string quantized_conv2d_with_bias_and_relu_and_requantize; + string quantized_concatv2; + string quantized_max_pool; + string quantized_conv2d_with_bias_sum_and_relu; + string quantized_conv2d_with_bias_sum_and_relu_and_requantize; + string quant_conv2d_with_bias_signed_sum_and_relu_and_requantize; string relu; string relu_grad; + string requantize; string tanh; string tanh_grad; string reshape; @@ -2833,6 +2939,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // // @return RewriteInfo* for the applicable rewrite rule const RewriteInfo* CheckForNodeRewrite(const Node* n) const; + const RewriteInfo* CheckForQuantizedNodeRewrite(const Node* n) const; // Default rewrite rule to be used in scenario 1 for rewrite. // @return - true (since we want to always rewrite) @@ -3091,7 +3198,11 @@ class MklLayoutRewritePass : public GraphOptimizationPass { static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb); + static void CopyAttrsQuantizedPooling(const Node* orig_node, NodeBuilder* nb); + static void CopyAttrsQuantizedConv2D(const Node* orig_node, NodeBuilder* nb); + static void CopyAttrsQuantizedConcat(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb); + static void CopyAttrsRequantize(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb); // Generate a graph node in graph 'g' representing a dummy Mkl tensor node, @@ -3411,8 +3522,27 @@ Status MklLayoutRewritePass::SetUpInputs( // We add workspace edge only for MaxPool, LRN and BatchNorm. std::vector workspace_tensors; bool are_workspace_tensors_available = false; - AddWorkSpaceEdgeIfNeeded(g, old_node, nb, &workspace_tensors, - &are_workspace_tensors_available); + + // Avoid workspace check for QuantizedConv2D and the fused + // Ops as they don't have attribute: "T". + std::vector quant_ops { + "QuantizedConv2D", + "QuantizedConv2DWithBias", + "QuantizedConv2DAndRelu", + "QuantizedConv2DWithBiasAndRelu", + "QuantizedConv2DWithBiasSumAndRelu", + "QuantizedConv2DAndRequantize", + "QuantizedConv2DWithBiasAndRequantize", + "QuantizedConv2DAndReluAndRequantize", + "QuantizedConv2DWithBiasAndReluAndRequantize", + "QuantizedConv2DWithBiasSumAndReluAndRequantize", + "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"}; + bool should_check_workspace = + std::find(std::begin(quant_ops), std::end(quant_ops), + old_node->type_string()) == std::end(quant_ops); + if (should_check_workspace) + AddWorkSpaceEdgeIfNeeded(g, old_node, nb, &workspace_tensors, + &are_workspace_tensors_available); int new_node_input_slots = 0; if (kTensorOrdering == MklTfTensorOrdering::TENSORS_INTERLEAVED) { @@ -3685,6 +3815,69 @@ void MklLayoutRewritePass::CopyAttrsDataType(const Node* orig_node, nb->Attr("T", T); } +void MklLayoutRewritePass::CopyAttrsQuantizedPooling(const Node* orig_node, + NodeBuilder* nb) { + DataType T; + string data_format; + string padding; + std::vector ksize, strides; + + // Get all attributes from old node. + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "ksize", &ksize)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding)); + + // Add attributes to new node. + nb->Attr("T", T); + nb->Attr("ksize", ksize); + nb->Attr("strides", strides); + nb->Attr("padding", padding); +} + +void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node, + NodeBuilder* nb) { + DataType Tinput, Tfilter, out_type; + string padding; + string data_format("NHWC"); + std::vector strides, dilations; + + // Get all attributes from old node. + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tinput", &Tinput)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tfilter", &Tfilter)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "out_type", &out_type)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations)); + + // Add attributes to new node. + nb->Attr("Tinput", Tinput); + nb->Attr("Tfilter", Tfilter); + nb->Attr("out_type", out_type); + nb->Attr("padding", padding); + nb->Attr("strides", strides); + nb->Attr("dilations", dilations); + nb->Attr("T", out_type); // added "T" for facilitating MklToTf conversion. + nb->Attr("data_format", data_format); + // Requantization attr Tbias + DataType Tbias; + Status bias_status = GetNodeAttr(orig_node->def(), "Tbias", &Tbias); + if (bias_status.ToString() == "OK") nb->Attr("Tbias", Tbias); +} + +void MklLayoutRewritePass::CopyAttrsRequantize(const Node* orig_node, + NodeBuilder* nb) { + DataType Tinput, out_type; + + // Get all attributes from old node. + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tinput", &Tinput)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "out_type", &out_type)); + + // Add attributes to new node. + nb->Attr("Tinput", Tinput); + nb->Attr("out_type", out_type); +} + void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb) { DataType T; @@ -4145,9 +4338,16 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr* g, } ri->copy_attrs(const_cast(orig_node), &nb); - // Set the Mkl layer label for this op. - nb.Attr("_kernel", mkl_op_registry::kMklOpLabel); + // Set the Mkl layer label for this op. + if (DataTypeIsQuantized(orig_node->input_type(0)) || + DataTypeIsQuantized(orig_node->output_type(0))) { +#ifdef INTEL_MKL_QUANTIZED + nb.Attr("_kernel", mkl_op_registry::kMklQuantizedOpLabel); +#endif + } else { + nb.Attr("_kernel", mkl_op_registry::kMklOpLabel); + } // Finalize graph and get new node. Node* new_node = nullptr; TF_CHECK_OK(nb.Finalize(&**g, &new_node)); @@ -4193,10 +4393,38 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr* g, return Status::OK(); } +// TODO(mdfaijul): Is there any other elegent way to check for quantized ops +// having attributes other than "T"? +// Current implementation reflects only QuantizedConv2D and its fused Ops. +const MklLayoutRewritePass::RewriteInfo* +MklLayoutRewritePass::CheckForQuantizedNodeRewrite(const Node* n) const { +#ifdef INTEL_MKL_QUANTIZED + DataType Tinput, Tfilter; + if (!(GetNodeAttr(n->def(), "Tinput", &Tinput).ok() && + GetNodeAttr(n->def(), "Tfilter", &Tfilter).ok())) { + return nullptr; + } + if (mkl_op_registry::IsMklOp(mkl_op_registry::GetMklOpName(n->type_string()), + Tinput, Tfilter)) { + for (auto ri = rinfo_.cbegin(); ri != rinfo_.cend(); ++ri) { + if (n->type_string().compare(ri->name) == 0 && ri->rewrite_rule(n)) { + return &*ri; + } + } + } +#endif + return nullptr; +} + const MklLayoutRewritePass::RewriteInfo* MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const { CHECK_NOTNULL(n); + // QuntizedOps may have attributes other than "T", so decoupled the check + // with a function, CheckForQuantizedNodeRewrite(const Node*). + const RewriteInfo* ri = CheckForQuantizedNodeRewrite(n); + if (ri != nullptr) return ri; + // First check if node along with its type is supported by MKL layer. // We do not want to rewrite an op into Mkl op if types are not supported. // E.g., MklRelu does not support INT32. So we cannot rewrite Relu to diff --git a/tensorflow/core/graph/mkl_tfconversion_pass.cc b/tensorflow/core/graph/mkl_tfconversion_pass.cc index aa39af637f..1530593fc9 100644 --- a/tensorflow/core/graph/mkl_tfconversion_pass.cc +++ b/tensorflow/core/graph/mkl_tfconversion_pass.cc @@ -146,22 +146,20 @@ Status MklToTfConversionPass::InsertConversionNodeOnEdge( CHECK_NOTNULL(dst); Node* conversion_node = nullptr; - DataType src_datatype = DT_INVALID; - DataType dst_datatype = DT_INVALID; + DataType src_datatype = src->output_type(e->src_output()); + DataType dst_datatype = dst->input_type(e->dst_input()); string data_format; - TF_CHECK_OK(GetNodeAttr(src->def(), "T", &src_datatype)); - bool dst_dtype_found = - GetNodeAttr(dst->def(), "T", &dst_datatype) == Status::OK(); // We compare source and destination datatypes only when both are found. - if (dst_dtype_found && (src_datatype != dst_datatype)) { - string err_msg = "T attribute of " + src->name() + " and " + dst->name() + - " do not match. Will not insert" + - " MklToTf node in such case."; + if (src_datatype != dst_datatype) { + string err_msg = "T attribute of " + src->name() + ":" + + std::to_string(e->src_output()) + " and " + dst->name() + + ":" + std::to_string(e->dst_input()) + + " do not" + " match. Will not insert MklToTf node in such case."; return Status(error::Code::INVALID_ARGUMENT, err_msg.c_str()); } - // Build the conversion node and specify src as input. TF_CHECK_OK( NodeBuilder((*g)->NewName("Mkl2Tf"), "_MklToTf") .Input(src, e->src_output()) diff --git a/tensorflow/core/kernels/mkl_tfconv_op.h b/tensorflow/core/kernels/mkl_tfconv_op.h index f4f0035f26..1e156fa531 100644 --- a/tensorflow/core/kernels/mkl_tfconv_op.h +++ b/tensorflow/core/kernels/mkl_tfconv_op.h @@ -201,6 +201,7 @@ class MklToTfOp : public OpKernel { MklToTfOp); TF_CALL_NUMBER_TYPES(REGISTER_CPU); +TF_CALL_QUANTIZED_TYPES(REGISTER_CPU); #undef REGISTER_CPU } // namespace tensorflow #endif // INTEL_MKL -- GitLab From 9d45b84c12c8c9fb7a928adb9affaa91b35c7e2b Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Mon, 13 Aug 2018 18:04:38 -0700 Subject: [PATCH 0020/1085] Ran clang-format tool --- tensorflow/tools/graph_transforms/BUILD | 1 + .../fuse_quantized_convolution.cc | 220 +++++++++++ .../tools/quantization/quantize_graph.py | 349 +++++++++++++++++- 3 files changed, 557 insertions(+), 13 deletions(-) create mode 100644 tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD index 1ad1895269..eb1ed1f2ca 100644 --- a/tensorflow/tools/graph_transforms/BUILD +++ b/tensorflow/tools/graph_transforms/BUILD @@ -97,6 +97,7 @@ cc_library( "fold_old_batch_norms.cc", "freeze_requantization_ranges.cc", "fuse_convolutions.cc", + "fuse_quantized_convolution.cc", "insert_logging.cc", "obfuscate_names.cc", "quantize_nodes.cc", diff --git a/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc b/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc new file mode 100644 index 0000000000..2128bcd978 --- /dev/null +++ b/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc @@ -0,0 +1,220 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef INTEL_MKL +#include + +#include "tensorflow/core/common_runtime/constant_folding.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/numeric_types.h" +#include "tensorflow/core/graph/graph_constructor.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/graph/subgraph.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/util/command_line_flags.h" +#include "tensorflow/tools/graph_transforms/fold_constants_lib.h" +#include "tensorflow/tools/graph_transforms/transform_utils.h" + +namespace tensorflow { +namespace graph_transforms { + +Status FuseQuantizedConvolutionAndRequantize( + const GraphDef& input_graph_def, const TransformFuncContext& context, + GraphDef* output_graph_def) { + std::map node_map; + MapNamesToNodes(input_graph_def, &node_map); + GraphDef replaced_graph_def; + TF_RETURN_IF_ERROR(ReplaceMatchingOpTypes( + input_graph_def, // clang-format off + + {"Requantize", + { + {"QuantizedConv2D|QuantizedConv2DWithBias|QuantizedConv2DWithRelu|" + "QuantizedConv2DWithBiasAndRelu|QuantizedConv2DWithBiasSumAndRelu"}, + {"QuantizedConv2D|QuantizedConv2DWithBias|QuantizedConv2DWithRelu|" + "QuantizedConv2DWithBiasAndRelu|QuantizedConv2DWithBiasSumAndRelu"}, + {"QuantizedConv2D|QuantizedConv2DWithBias|QuantizedConv2DWithRelu|" + "QuantizedConv2DWithBiasAndRelu|QuantizedConv2DWithBiasSumAndRelu"}, + {"Const"}, + {"Const"} + } + }, // clang-format on */ + [&node_map](const NodeMatch& match, const std::set& input_nodes, + const std::set& output_nodes, + std::vector* new_nodes) { + // TODO(mdfaijul/sheng): Current implementation assumed all + // requantization cases have bias. Index of inputs need to be updated + // for non-bias cases. + + // Find all the nodes we expect in the subgraph. + const NodeDef& requantize_node = match.node; + CHECK_EQ("Requantize", requantize_node.op()); + const NodeDef& quantized_conv2D_node = match.inputs[0].node; + const NodeDef& const_requantize_range_min_node = match.inputs[3].node; + CHECK_EQ("Const", const_requantize_range_min_node.op()); + const NodeDef& const_requantize_range_max_node = match.inputs[4].node; + CHECK_EQ("Const", const_requantize_range_max_node.op()); + + string quantized_conv2D_op_name = quantized_conv2D_node.op(); + // Set up the new fused version of the convolution op. + NodeDef fused_conv; + fused_conv.set_op(quantized_conv2D_op_name + "AndRequantize"); + fused_conv.set_name(match.node.name()); + int n_input = quantized_conv2D_node.input_size(); + if (quantized_conv2D_op_name.compare( + "QuantizedConv2DWithBiasSumAndRelu") == 0) + n_input -= 1; // -1 since summand is moved after frozen min-max + + for (int i=0; i < n_input; i++) + AddNodeInput(quantized_conv2D_node.input(i), &fused_conv); + + AddNodeInput(const_requantize_range_min_node.name(), &fused_conv); + AddNodeInput(const_requantize_range_max_node.name(), &fused_conv); + + // Add additional inputs to + // QuantizedConv2DWithBiasSumAndReluAndRequantize + if (quantized_conv2D_op_name.compare( + "QuantizedConv2DWithBiasSumAndRelu") == 0) { + const NodeDef *in_requantize = node_map[node_map[ + quantized_conv2D_node.input(n_input)]->input(0)]; + string summand(in_requantize->name()); + string min_summand(in_requantize->name() + ":1"); + string max_summand(in_requantize->name() + ":2"); + AddNodeInput(summand, &fused_conv); + AddNodeInput(min_summand, &fused_conv); + AddNodeInput(max_summand, &fused_conv); + + // Signed version QuantizedConv2DWithBiasSumAndReluAndRequantize + // if Relu does not follow the convolution operation + std::vector signed_ops = { + "QuantizedConv2DWithBias", + "QuantizedConv2D" + }; + bool is_signed_summand = + std::find(signed_ops.begin(), signed_ops.end(), + node_map[in_requantize->input(0)]->op()) != signed_ops.end(); + if (is_signed_summand) { + fused_conv.set_op( + "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"); + SetNodeAttr("Tsummand", DT_QINT8, &fused_conv); + } else { + SetNodeAttr("Tsummand", DT_QUINT8, &fused_conv); + } + } + CopyNodeAttr(quantized_conv2D_node, "Tinput", "Tinput", &fused_conv); + CopyNodeAttr(quantized_conv2D_node, "Tfilter", "Tfilter", &fused_conv); + CopyNodeAttr(quantized_conv2D_node, "strides", "strides", &fused_conv); + CopyNodeAttr(quantized_conv2D_node, "padding", "padding", &fused_conv); + + // Copy dilation attribute if exsit in the orginal node + if (HasNodeAttr(quantized_conv2D_node, "dilations")) + CopyNodeAttr(quantized_conv2D_node, "dilations", + "dilations", &fused_conv); + if (quantized_conv2D_op_name.compare("QuantizedConv2D") == 0 || + quantized_conv2D_op_name.compare("QuantizedConv2DWithBias") == 0) + SetNodeAttr("out_type", DT_QINT8, &fused_conv); + else + SetNodeAttr("out_type", DT_QUINT8, &fused_conv); + new_nodes->push_back(fused_conv); + new_nodes->push_back(const_requantize_range_min_node); + new_nodes->push_back(const_requantize_range_max_node); + + return Status::OK(); + }, + {}, &replaced_graph_def)); + + // Convert bias float -> int32 on replaced_graph_def + std::vector fused_requantized_bias_ops = { + "QuantizedConv2DWithBiasAndRequantize", + "QuantizedConv2DWithBiasAndReluAndRequantize", + "QuantizedConv2DWithBiasSumAndReluAndRequantize", + "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize" + }; + node_map.clear(); + MapNamesToNodes(replaced_graph_def, &node_map); + for (auto& node_pair : node_map) { + const NodeDef *node = node_pair.second; + bool is_fused_requantized_conv_op = + std::find(fused_requantized_bias_ops.begin(), + fused_requantized_bias_ops.end(), + node->op()) != fused_requantized_bias_ops.end(); + if (is_fused_requantized_conv_op) { + // If the op is not fed by Another Requantize op, + // then we coonvert bias as Int32 + string input_op = node_map[NodeNameFromInput(node->input(0))]->op(); + if (str_util::StartsWith(input_op, "QuantizedConv2D") && + str_util::EndsWith(input_op, "AndRequantize")) { + NodeDef *bias_node = const_cast(node_map[NodeNameFromInput( + node->input(2))]); + const NodeDef *min_input_node = node_map[NodeNameFromInput( + node_map[node->input(0)]->input(7))]; + const NodeDef *max_input_node = node_map[NodeNameFromInput( + node_map[node->input(0)]->input(8))]; + const NodeDef *min_filter_node = node_map[NodeNameFromInput( + node->input(5))]; + const NodeDef *max_filter_node = node_map[NodeNameFromInput( + node->input(6))]; + const float min_input = + GetNodeTensorAttr(*min_input_node, "value").flat()(0); + const float max_input = + GetNodeTensorAttr(*max_input_node, "value").flat()(0); + const float min_filter = + GetNodeTensorAttr(*min_filter_node, "value").flat()(0); + const float max_filter = + GetNodeTensorAttr(*max_filter_node, "value").flat()(0); + + TensorProto float_tensor_proto = bias_node->attr().at("value").tensor(); + Tensor float_tensor; + CHECK(float_tensor.FromProto(float_tensor_proto)); + CHECK_EQ(float_tensor.dtype(), DT_FLOAT); + float *p_bias_float = float_tensor.flat().data(); + + Tensor int32_tensor = Tensor(DT_QINT32, float_tensor.shape()); + qint32 *p_bias_int32 = int32_tensor.flat().data(); + + float bias_scale = 255.0 * 127.0 / + (std::max(std::abs(max_input), std::abs(min_input)) * + std::max(std::abs(max_filter), std::abs(min_filter))); + int64 nelems = float_tensor.NumElements(); + for (int64 n = 0; n < nelems; n++) + p_bias_int32[n] = (int32_t) (p_bias_float[n] * bias_scale); + + bias_node->clear_attr(); + AttrValue attr_type; + attr_type.set_type(int32_tensor.dtype()); + bias_node->mutable_attr()->insert({"dtype", attr_type}); + + AttrValue attr_tensor; + TensorProto* t = attr_tensor.mutable_tensor(); + int32_tensor.AsProtoTensorContent(t); + bias_node->mutable_attr()->insert({"value", attr_tensor}); + SetNodeAttr("Tbias", DT_QINT32, const_cast(node)); + } else { + SetNodeAttr("Tbias", DT_FLOAT, const_cast(node)); + } + } + } + *output_graph_def = replaced_graph_def; + return Status::OK(); +} + +REGISTER_GRAPH_TRANSFORM("fuse_quantized_conv_and_requantize", + FuseQuantizedConvolutionAndRequantize); + +} // namespace graph_transforms +} // namespace tensorflow +#endif // INTEL_MKL diff --git a/tensorflow/tools/quantization/quantize_graph.py b/tensorflow/tools/quantization/quantize_graph.py index 3acb532263..14b572c15f 100644 --- a/tensorflow/tools/quantization/quantize_graph.py +++ b/tensorflow/tools/quantization/quantize_graph.py @@ -21,6 +21,7 @@ bazel build tensorflow/tools/quantization:quantize_graph \ --output_node_names="softmax2" --print_nodes --output=/tmp/quantized_graph.pb \ --mode=eightbit --logtostderr +To quantize for Intel CPU, add --intel_cpu_eightbitize=True. """ from __future__ import absolute_import @@ -46,6 +47,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.platform import app from tensorflow.python.platform import flags as flags_lib from tensorflow.python.platform import gfile +from google.protobuf import text_format flags = flags_lib FLAGS = flags.FLAGS @@ -87,7 +89,14 @@ flags.DEFINE_float( "information. Note: this should be considered a coarse tool just good " "enough for experimentation purposes, since graphs quantized in this way " "would be very inaccurate.") - +flags.DEFINE_boolean("input_binary", True, + """Input graph binary or text.""") +flags.DEFINE_boolean("output_binary", True, + """Output graph binary or text.""") +flags.DEFINE_boolean( + "intel_cpu_eightbitize", False, + "If true eightbitized graph will include fused quantized" + "nodes in the output_graph for Intel CPU.") def print_input_nodes(current_node, nodes_map, indent, already_visited): print(" " * indent + current_node.op + ":" + current_node.name) @@ -297,6 +306,8 @@ def quantize_weight_eightbit(input_node, quantization_mode): dtypes.quint8, mode=quantization_mode) quint8_tensor = quantize_op[0].eval() + min_value = quantize_op[1].eval() + max_value = quantize_op[2].eval() shape = tensor_util.TensorShapeProtoToList(input_node.attr["value"] .tensor.tensor_shape) quint8_const_node = create_constant_node( @@ -309,6 +320,57 @@ def quantize_weight_eightbit(input_node, quantization_mode): set_attr_string(dequantize_node, "mode", quantization_mode) return [quint8_const_node, min_node, max_node, dequantize_node] +# TODO(intel-tf): Current Intel-CPU quantized Conv2D and Matmul supports only +# signed scaled mode of weight quantization. +def intel_cpu_quantize_weight_eightbit(input_node, quantization_mode="SCALED"): + """Returns replacement of constant weight node. + + This function creates (i) a quantized constant node, (ii) a float min node + (iii) a float max node, and (iv) a dequantize node.""" + base_name = input_node.name + "_" + qint8_const_name = base_name + "qint8_const" + min_name = base_name + "min" + max_name = base_name + "max" + float_tensor = tensor_util.MakeNdarray(input_node.attr["value"].tensor) + min_value = np.min(float_tensor.flatten()) + max_value = np.max(float_tensor.flatten()) + # Same processing of min-max as in quantize_weight_eightbit function. + if min_value > 0.0: + min_value = 0.0 + if min_value == max_value: + if abs(min_value) < 0.000001: + max_value = min_value + 1.0 + elif min_value > 0: + max_value = 2 * min_value + else: + max_value = min_value / 2.0 + + sess = session.Session() + with sess.as_default(): + quantize_op = array_ops.quantize_v2( + float_tensor, + min_value, + max_value, + dtypes.qint8, + mode=quantization_mode, + round_mode="HALF_TO_EVEN") + qint8_tensor = quantize_op[0].eval() + # Updated min-max values should be passed to the next feeding node. + min_value = quantize_op[1].eval() + max_value = quantize_op[2].eval() + shape = tensor_util.TensorShapeProtoToList(input_node.attr["value"] + .tensor.tensor_shape) + qint8_const_node = create_constant_node( + qint8_const_name, qint8_tensor, + dtypes.qint8, + shape=shape) + min_node = create_constant_node(min_name, min_value, dtypes.float32) + max_node = create_constant_node(max_name, max_value, dtypes.float32) + dequantize_node = create_node("Dequantize", input_node.name, + [qint8_const_name, min_name, max_name]) + set_attr_dtype(dequantize_node, "T", dtypes.qint8) + set_attr_string(dequantize_node, "mode", b'SCALED') + return [qint8_const_node, min_node, max_node, dequantize_node] EightbitizeRecursionState = collections.namedtuple( "EightbitizeRecursionState", @@ -322,7 +384,8 @@ class GraphRewriter(object): input_graph, mode, quantized_input_range, - fallback_quantization_range=None): + fallback_quantization_range=None, + intel_cpu_eightbitize=False): """Sets up the class to rewrite a float graph. Args: @@ -344,6 +407,7 @@ class GraphRewriter(object): self.nodes_map = self.create_nodes_map(input_graph) self.output_graph = None self.mode = mode + self.intel_cpu_eightbitize = intel_cpu_eightbitize self.final_node_renames = {} if quantized_input_range: self.input_range = (quantized_input_range[0], quantized_input_range[1]) @@ -417,8 +481,22 @@ class GraphRewriter(object): self.state = EightbitizeRecursionState( already_visited={}, output_node_stack=[], merged_with_fake_quant={}) - for output_node in output_nodes: - self.eightbitize_nodes_recursively(output_node) + + if self.intel_cpu_eightbitize: + # TODO(intel-tf): Enables fused quantized node for intel cpu. + for output_node in output_nodes: + # Intiailize output_node_stack with output node. + # Each element in the stack is a mutable list containing + # [parent_node, index_to_parent, quantization_flag, fusion_flag]. + # In case of root node, make self as parent. + self.state.output_node_stack.append( + [output_node, None, False, False]) + self.intel_cpu_eightbitize_nodes_recursively(output_node) + self.state.output_node_stack.pop() + else: + for output_node in output_nodes: + self.eightbitize_nodes_recursively(output_node) + self.state = None if self.input_range: self.add_output_graph_node( @@ -653,6 +731,200 @@ class GraphRewriter(object): (self.state.output_node_stack[-1][0], current_node.name, current_node.op)) + # TODO(intel-tf): Quantized Conv2D could be fused with few other succeeding + # ops. Current support is for BiasAdd and Relu. Future implementation will + # include: + # (i) Conv2D + {BiasAdd} + Relu + Add + Relu + # (ii) Conv2D + {BiasAdd} + Relu + Add + # (ii) Conv2D + {BiasAdd} + Add + Relu + # (iii) Conv2D + {BiasAdd} + Add + def intel_cpu_eightbitize_conv_node(self, original_node, bias_node=None, + bias_add_name=None, add_node_name=None, + relu_node_name=None): + """Replaces a Conv2D node with the eight bit equivalent sub-graph.""" + all_input_names = self.add_eightbit_prologue_nodes(original_node) + + if bias_node and add_node_name and relu_node_name: + new_node = node_def_pb2.NodeDef() + new_node.CopyFrom(bias_node) + self.add_output_graph_node(new_node) + all_input_names = all_input_names[:2] + [bias_node.name] + \ + all_input_names[2:] + [add_node_name] + quantized_conv_name = original_node.name + "_eightbit_quantized_conv" + quantized_conv_node = create_node("QuantizedConv2DWithBiasSumAndRelu", + quantized_conv_name, all_input_names) + elif bias_node and (not add_node_name) and relu_node_name: + new_node = node_def_pb2.NodeDef() + new_node.CopyFrom(bias_node) + self.add_output_graph_node(new_node) + all_input_names = all_input_names[:2] + [bias_node.name] + \ + all_input_names[2:] + quantized_conv_name = original_node.name + "_eightbit_quantized_conv" + quantized_conv_node = create_node("QuantizedConv2DWithBiasAndRelu", + quantized_conv_name, all_input_names) + elif bias_node and bias_add_name and \ + (not add_node_name) and (not relu_node_name): + new_node = node_def_pb2.NodeDef() + new_node.CopyFrom(bias_node) + self.add_output_graph_node(new_node) + all_input_names = all_input_names[:2] + [bias_node.name] + \ + all_input_names[2:] + quantized_conv_name = original_node.name + "_eightbit_quantized_conv" + quantized_conv_node = create_node("QuantizedConv2DWithBias", + quantized_conv_name, all_input_names) + else: + quantized_conv_name = original_node.name + "_eightbit_quantized_conv" + quantized_conv_node = create_node("QuantizedConv2D", quantized_conv_name, + all_input_names) + copy_attr(quantized_conv_node, "strides", original_node.attr["strides"]) + copy_attr(quantized_conv_node, "padding", original_node.attr["padding"]) + copy_attr(quantized_conv_node, "dilations", original_node.attr["dilations"]) + set_attr_dtype(quantized_conv_node, "Tinput", dtypes.quint8) + set_attr_dtype(quantized_conv_node, "Tfilter", dtypes.qint8) + set_attr_dtype(quantized_conv_node, "out_type", dtypes.qint32) + self.add_output_graph_node(quantized_conv_node) + quantize_down_name = self.add_quantize_down_nodes(original_node, + quantized_conv_name) + if bias_node and relu_node_name: + self.add_dequantize_result_node(quantize_down_name, relu_node_name) + elif bias_node and bias_add_name and \ + (not add_node_name) and (not relu_node_name): + self.add_dequantize_result_node(quantize_down_name, bias_add_name) + else: + self.add_dequantize_result_node(quantize_down_name, original_node.name) + + # TODO(intel-tf): To check whether Conv2D is fed by relu directly or via + # pooling ops. This is required as intel cpu requires input tensor for Conv2D + # to be non-negative. + def intel_cpu_find_relu_recursively(self, current_node): + """Helper function to check if Conv2D is fed by Relu.""" + if current_node.op == "Relu": + return True + else: + first_input_node_name = node_name_from_input(current_node.input[0]) + input_node = self.nodes_map[first_input_node_name] + if input_node.op in ("ConcatV2", "MaxPool", "AvgPool", "Relu"): + return self.intel_cpu_find_relu_recursively(input_node) + else: + return False + + # TODO(intel-tf): We leave the output graph partially quantized for + # intel cpu. Current quantization support is for Conv2D and its fusion. + # More quantized operations will be included as more implementations are + # completed. + def intel_cpu_eightbitize_nodes_recursively(self, current_node): + """The entry point for transforming a graph into full eight bit.""" + if current_node.name in self.state.already_visited: + if (self.should_merge_with_fake_quant_node() or + current_node.name in self.state.merged_with_fake_quant): + raise ValueError("Unsupported graph structure: output of node %s " + "is processed by a FakeQuant* node and should have " + "no other outputs.", current_node.name) + return + + self.state.already_visited[current_node.name] = True + quantize_input, should_quantize_conv, \ + fuse_with_conv = (False, False, False) + + if current_node.op == "Conv2D": + should_quantize_conv = self.intel_cpu_find_relu_recursively(current_node) + + inputs = list(enumerate(current_node.input)) + if current_node.op == "AddN": + inputs = reversed(inputs) # pylint: disable=redefined-variable-type + + for i, input_node_name in inputs: + input_node_name = node_name_from_input(input_node_name) + input_node = self.nodes_map[input_node_name] + + if should_quantize_conv and i == 1 and input_node.op == "Const": + quantize_input = True + + self.state.output_node_stack.append([current_node, i, quantize_input, + fuse_with_conv]) + self.intel_cpu_eightbitize_nodes_recursively(input_node) + self.state.output_node_stack.pop() + + if current_node.op == "Conv2D" and should_quantize_conv and quantize_input: + # match pattern for fusion with bias and relu + grand_parent, parent = self.state.output_node_stack[-2:] + if parent[0].op == "BiasAdd" and grand_parent[0].op == "Relu": + self.state.output_node_stack[-2][3] = True # BiasAdd to be fused + self.state.output_node_stack[-3][3] = True # Relu to be fused + bias_node_name = node_name_from_input(parent[0].input[1]) + bias_node = self.nodes_map[bias_node_name] + self.intel_cpu_eightbitize_conv_node(current_node, bias_node, None, + None, grand_parent[0].name) + elif parent[0].op == "BiasAdd" and grand_parent[0].op == "AddN": + grand_grand_parent = self.state.output_node_stack[-3] + if grand_grand_parent[0].op == "Relu" \ + and (not self.state.output_node_stack[-3][3]) \ + and (not self.state.output_node_stack[-4][3]): + self.state.output_node_stack[-2][3] = True # BiasAdd to be fused + self.state.output_node_stack[-3][3] = True # AddN to be fused + self.state.output_node_stack[-4][3] = True # Relu to be fused + bias_node_name = node_name_from_input(parent[0].input[1]) + bias_node = self.nodes_map[bias_node_name] + add_node_name = node_name_from_input(grand_parent[0].input[0]) + self.intel_cpu_eightbitize_conv_node(current_node, bias_node, None, + add_node_name, + grand_grand_parent[0].name) + elif not self.state.output_node_stack[-2][3]: # Fuse BiasAdd then + self.state.output_node_stack[-2][3] = True # BiasAdd to be fused + bias_node_name = node_name_from_input(parent[0].input[1]) + bias_node = self.nodes_map[bias_node_name] + self.intel_cpu_eightbitize_conv_node(current_node, bias_node, + parent[0].name) + else: + self.intel_cpu_eightbitize_conv_node(current_node) + elif parent[0].op == "BiasAdd" and \ + (not self.state.output_node_stack[-2][3]): + self.state.output_node_stack[-2][3] = True # BiasAdd to be fused + bias_node_name = node_name_from_input(parent[0].input[1]) + bias_node = self.nodes_map[bias_node_name] + self.intel_cpu_eightbitize_conv_node(current_node, bias_node, + parent[0].name) + else: + self.intel_cpu_eightbitize_conv_node(current_node) + elif current_node.op == "BiasAdd" and \ + self.state.output_node_stack[-1][3]: + pass # This op is already processed by fused quantization + elif current_node.op == "Relu" and \ + self.state.output_node_stack[-1][3]: + pass # This op is already processed by fused quantization + elif current_node.op == "AddN" and \ + self.state.output_node_stack[-1][3]: + pass # AddN op is already processed by fused quatization + elif current_node.op == "MaxPool" or current_node.op == "AvgPool": + self.eightbitize_single_input_tensor_node(current_node, + self.add_pool_function) + elif (current_node.op == "ConcatV2" and + dtypes.as_dtype(current_node.attr["T"].type) == dtypes.float32): + self.eightbitize_concatv2_node(current_node) + elif current_node.op == "Const": + parent = self.state.output_node_stack[-1] + if parent[0].op == "Conv2D" and parent[2]: + for n in intel_cpu_quantize_weight_eightbit(current_node, b"SCALED"): + self.add_output_graph_node(n) + elif parent[0].op == "BiasAdd" and \ + self.state.output_node_stack[-2][3]: + pass # This constant is already process by fused quantization + else: + new_node = node_def_pb2.NodeDef() + new_node.CopyFrom(current_node) + self.add_output_graph_node(new_node) + else: + new_node = node_def_pb2.NodeDef() + new_node.CopyFrom(current_node) + self.add_output_graph_node(new_node) + + if (self.should_merge_with_fake_quant_node() and + current_node.name not in self.state.merged_with_fake_quant): + raise ValueError( + "FakeQuant* node %s failed to merge with node %s of type %s" % + (self.state.output_node_stack[-1][0], current_node.name, + current_node.op)) + def add_eightbit_prologue_nodes(self, original_node): """Adds input conversion nodes to handle quantizing the underlying node.""" namespace_prefix = original_node.name + "_eightbit" @@ -712,7 +984,11 @@ class GraphRewriter(object): "QuantizeV2", quantize_input_name, [original_input_name, min_input_name, max_input_name]) set_attr_dtype(quantize_input_node, "T", dtypes.quint8) - set_attr_string(quantize_input_node, "mode", b"MIN_FIRST") + set_attr_string(quantize_input_node, "mode", + b"SCALED" if self.intel_cpu_eightbitize else b"MIN_FIRST") + set_attr_string(quantize_input_node, "round_mode", + b"HALF_TO_EVEN" if self.intel_cpu_eightbitize + else b"HALF_AWAY_FROM_ZERO") self.add_output_graph_node(quantize_input_node) min_output_name = quantize_input_name + ":1" max_output_name = quantize_input_name + ":2" @@ -965,6 +1241,44 @@ class GraphRewriter(object): self.add_output_graph_node(quantized_concat_node) self.add_dequantize_result_node(quantized_concat_name, original_node.name) + def eightbitize_concatv2_node(self, original_node): + """ + Args: + original_node: Float node to be converted. + + Returns: + Subgraph representing the quantized version of the original node. + + """ + namespace_prefix = original_node.name + "_eightbit" + quantized_concat_name = namespace_prefix + "_quantized_concatv2" + reshape_dims_name, reduction_dims_name = self.add_common_quantization_nodes( + namespace_prefix) + num_input = len(original_node.input) + shape_input_name = original_node.input[num_input-1] + original_inputs = original_node.input[0:num_input-1] + input_names = [] + min_names = [] + max_names = [] + for original_input_name in original_inputs: + quantize_input_name, min_input_name, max_input_name = ( + self.eightbitize_input_to_node(namespace_prefix, original_input_name, + reshape_dims_name, + reduction_dims_name)) + input_names.append(quantize_input_name) + min_names.append(min_input_name) + max_names.append(max_input_name) + all_input_names = input_names + all_input_names.append(shape_input_name) + all_input_names.extend(min_names) + all_input_names.extend(max_names) + quantized_concat_node = create_node("QuantizedConcatV2", + quantized_concat_name, all_input_names) + set_attr_int(quantized_concat_node, "N", len(original_inputs)) + set_attr_dtype(quantized_concat_node, "T", dtypes.quint8) + self.add_output_graph_node(quantized_concat_node) + self.add_dequantize_result_node(quantized_concat_name, original_node.name) + def eightbitize_placeholder_node(self, current_node): """Replaces a placeholder node with a quint8 placeholder node+dequantize.""" name = current_node.name @@ -1249,7 +1563,6 @@ class GraphRewriter(object): self.input_graph = new_input_graph self.nodes_map = self.create_nodes_map(self.input_graph) - def main(unused_args): if not gfile.Exists(FLAGS.input): print("Input graph file '" + FLAGS.input + "' does not exist!") @@ -1264,9 +1577,14 @@ def main(unused_args): return -1 tf_graph = graph_pb2.GraphDef() - with gfile.Open(FLAGS.input, "rb") as f: + # TODO(intel-tf): Enabling user to work with both binary and text format. + mode = "rb" if FLAGS.input_binary else "r" + with gfile.Open(FLAGS.input, mode) as f: data = f.read() - tf_graph.ParseFromString(data) + if FLAGS.input_binary: + tf_graph.ParseFromString(data) + else: + text_format.Merge(data, tf_graph) graph = ops.Graph() with graph.as_default(): @@ -1287,16 +1605,21 @@ def main(unused_args): FLAGS.quantized_fallback_min, FLAGS.quantized_fallback_max ] - rewriter = GraphRewriter(tf_graph, FLAGS.mode, quantized_input_range, - fallback_quantization_range) + rewriter = GraphRewriter(tf_graph, FLAGS.mode, + quantized_input_range, fallback_quantization_range, + FLAGS.intel_cpu_eightbitize) output_graph = rewriter.rewrite(FLAGS.output_node_names.split(",")) - f = gfile.FastGFile(FLAGS.output, "wb") - f.write(output_graph.SerializeToString()) + # TODO(intel-tf): Enabling user to work with both binary and text format. + mode = "wb" if FLAGS.output_binary else "w" + f = gfile.FastGFile(FLAGS.output, mode) + if FLAGS.output_binary: + f.write(output_graph.SerializeToString()) + else: + f.write(str(output_graph)) return 0 - if __name__ == "__main__": app.run() -- GitLab From aa25cc078c9b55e5ca3e0f59df43e169bfee8f3c Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Thu, 16 Aug 2018 19:04:37 +0800 Subject: [PATCH 0021/1085] Add LeakyRelu C++ Op and its gradient implementation. LeakyRelu, defined as 'y = { x (x>=0) or alpha*x (x<0) }', was computed by combined Ops 'max(x, alpha*x)' in current codes. Hence its gradient calculation for back propagation would contain a serial of element-wise Ops. This looks really unnecessary for such a simple op and it could be done within just one Op with less memory accesses. --- tensorflow/cc/gradients/nn_grad.cc | 13 ++ tensorflow/cc/gradients/nn_grad_test.cc | 13 ++ tensorflow/core/kernels/relu_op.cc | 153 +++++++++++------- tensorflow/core/kernels/relu_op.h | 59 +++++++ tensorflow/core/kernels/relu_op_functor.h | 31 ++++ tensorflow/core/kernels/relu_op_gpu.cu.cc | 18 ++- tensorflow/core/ops/nn_ops.cc | 15 ++ tensorflow/core/ops/ops.pbtxt | 68 ++++++++ tensorflow/python/eager/pywrap_tfe_src.cc | 2 + .../python/kernel_tests/relu_op_test.py | 113 +++++++++++++ tensorflow/python/ops/nn_grad.py | 15 ++ tensorflow/python/ops/nn_ops.py | 3 +- 12 files changed, 432 insertions(+), 71 deletions(-) diff --git a/tensorflow/cc/gradients/nn_grad.cc b/tensorflow/cc/gradients/nn_grad.cc index 588e96cb19..0fc23d0bf7 100644 --- a/tensorflow/cc/gradients/nn_grad.cc +++ b/tensorflow/cc/gradients/nn_grad.cc @@ -143,6 +143,19 @@ Status Relu6GradHelper(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("Relu6", Relu6GradHelper); +Status LeakyReluGradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + float alpha; + TF_RETURN_IF_ERROR(GetNodeAttr(op.node()->attrs(), "alpha", &alpha)); + internal::LeakyReluGrad::Attrs attrs; + attrs.Alpha(alpha); + auto dx = internal::LeakyReluGrad(scope, grad_inputs[0], op.input(0), attrs); + grad_outputs->push_back(dx); + return scope.status(); +} +REGISTER_GRADIENT_OP("LeakyRelu", LeakyReluGradHelper); + Status EluGradHelper(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index aa72cf7ba2..5ebece7b6e 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -41,6 +41,7 @@ using ops::MaxPoolV2; using ops::Placeholder; using ops::Relu; using ops::Relu6; +using ops::LeakyRelu; using ops::Selu; using ops::Softmax; using ops::Softplus; @@ -160,6 +161,18 @@ TEST_F(NNGradTest, Relu6Grad) { RunTest(x, x_init_value, y, shape); } +TEST_F(NNGradTest, LeakyReluGrad) { + TensorShape shape({5, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + auto y = LeakyRelu(scope_, x); + // Avoid input values where Leaky ReLU gradient is not well defined (around + // zero). + Tensor x_init_value = test::AsTensor( + {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, + {5, 2}); + RunTest(x, x_init_value, y, shape); +} + TEST_F(NNGradTest, EluGrad) { TensorShape shape({5, 2}); auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); diff --git a/tensorflow/core/kernels/relu_op.cc b/tensorflow/core/kernels/relu_op.cc index d52358737f..c4f2ef5632 100644 --- a/tensorflow/core/kernels/relu_op.cc +++ b/tensorflow/core/kernels/relu_op.cc @@ -33,19 +33,25 @@ typedef Eigen::GpuDevice GPUDevice; typedef Eigen::SyclDevice SYCLDevice; #endif // TENSORFLOW_USE_SYCL -#define REGISTER_RELU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu").Device(DEVICE_CPU).TypeConstraint("T"), \ - ReluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("ReluGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ - ReluGradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6").Device(DEVICE_CPU).TypeConstraint("T"), \ - Relu6Op); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6Grad").Device(DEVICE_CPU).TypeConstraint("T"), \ - Relu6GradOp) +#define REGISTER_RELU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu").Device(DEVICE_CPU).TypeConstraint("T"), \ + ReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("ReluGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ + ReluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6").Device(DEVICE_CPU).TypeConstraint("T"), \ + Relu6Op); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6Grad").Device(DEVICE_CPU).TypeConstraint("T"), \ + Relu6GradOp) \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyRelu").Device(DEVICE_CPU).TypeConstraint("T"), \ + LeakyReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyReluGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ + LeakyReluGradOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER_RELU_KERNELS); #undef REGISTER_RELU_KERNELS @@ -99,6 +105,19 @@ namespace functor { extern template struct Relu6Grad; \ \ template <> \ + void LeakyRelu::operator()( \ + const GPUDevice& d, typename TTypes::ConstTensor features, \ + T alpha, typename TTypes::Tensor activations); \ + extern template struct LeakyRelu; \ + \ + template <> \ + void LeakyReluGrad::operator()( \ + const GPUDevice& d, typename TTypes::ConstTensor gradients, \ + typename TTypes::ConstTensor features, \ + T alpha, typename TTypes::Tensor backprops); \ + extern template struct LeakyReluGrad; \ + \ + template <> \ void Elu::operator()(const GPUDevice& d, \ typename TTypes::ConstTensor features, \ typename TTypes::Tensor activations); \ @@ -128,30 +147,36 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPEC); } // namespace functor // Registration of the GPU implementations. -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu").Device(DEVICE_GPU).TypeConstraint("T"), \ - ReluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("ReluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ - ReluGradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6").Device(DEVICE_GPU).TypeConstraint("T"), \ - Relu6Op); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6Grad").Device(DEVICE_GPU).TypeConstraint("T"), \ - Relu6GradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Elu").Device(DEVICE_GPU).TypeConstraint("T"), \ - EluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("EluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ - EluGradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Selu").Device(DEVICE_GPU).TypeConstraint("T"), \ - SeluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("SeluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu").Device(DEVICE_GPU).TypeConstraint("T"), \ + ReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("ReluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ + ReluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6").Device(DEVICE_GPU).TypeConstraint("T"), \ + Relu6Op); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6Grad").Device(DEVICE_GPU).TypeConstraint("T"), \ + Relu6GradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyRelu").Device(DEVICE_GPU).TypeConstraint("T"), \ + LeakyReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyReluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ + LeakyReluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Elu").Device(DEVICE_GPU).TypeConstraint("T"), \ + EluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("EluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ + EluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Selu").Device(DEVICE_GPU).TypeConstraint("T"), \ + SeluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("SeluGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ SeluGradOp) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); @@ -161,30 +186,36 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); #ifdef TENSORFLOW_USE_SYCL // Registration of the GPU implementations. -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu").Device(DEVICE_SYCL).TypeConstraint("T"), \ - ReluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("ReluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ - ReluGradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6").Device(DEVICE_SYCL).TypeConstraint("T"), \ - Relu6Op); \ - REGISTER_KERNEL_BUILDER( \ - Name("Relu6Grad").Device(DEVICE_SYCL).TypeConstraint("T"), \ - Relu6GradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Elu").Device(DEVICE_SYCL).TypeConstraint("T"), \ - EluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("EluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ - EluGradOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("Selu").Device(DEVICE_SYCL).TypeConstraint("T"), \ - SeluOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("SeluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu").Device(DEVICE_SYCL).TypeConstraint("T"), \ + ReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("ReluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ + ReluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6").Device(DEVICE_SYCL).TypeConstraint("T"), \ + Relu6Op); \ + REGISTER_KERNEL_BUILDER( \ + Name("Relu6Grad").Device(DEVICE_SYCL).TypeConstraint("T"), \ + Relu6GradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyRelu").Device(DEVICE_SYCL).TypeConstraint("T"), \ + LeakyReluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("LeakyReluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ + LeakyReluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Elu").Device(DEVICE_SYCL).TypeConstraint("T"), \ + EluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("EluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ + EluGradOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("Selu").Device(DEVICE_SYCL).TypeConstraint("T"), \ + SeluOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("SeluGrad").Device(DEVICE_SYCL).TypeConstraint("T"), \ SeluGradOp) TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNELS); diff --git a/tensorflow/core/kernels/relu_op.h b/tensorflow/core/kernels/relu_op.h index e712b02bd7..c55190065c 100644 --- a/tensorflow/core/kernels/relu_op.h +++ b/tensorflow/core/kernels/relu_op.h @@ -131,6 +131,65 @@ void Relu6GradOp::OperateNoTemplate(OpKernelContext* context, output->flat()); } +template +class LeakyReluOp : public UnaryElementWiseOp> { + public: + explicit LeakyReluOp(OpKernelConstruction* context) + : UnaryElementWiseOp>(context) { + float alpha_tmp; + OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_tmp)); + alpha_ = T(alpha_tmp); + } + + void Operate(OpKernelContext* context, const Tensor& input, Tensor* output) { + functor::LeakyRelu functor; + functor(context->eigen_device(), input.flat(), + alpha_, output->flat()); + } + + private: + T alpha_; +}; + +template +class LeakyReluGradOp + : public BinaryElementWiseOp> { + public: + explicit LeakyReluGradOp(OpKernelConstruction* context) + : BinaryElementWiseOp>(context) { + float alpha_tmp; + OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_tmp)); + alpha_ = T(alpha_tmp); + } + + void OperateNoTemplate(OpKernelContext* context, const Tensor& g, + const Tensor& a, T alpha, Tensor* output); + + // INPUTS: + // g (gradients): backpropagated gradients + // a (inputs): either the inputs that were passed to LeakyReluOp(), or its + // outputs (using either one yields the same result here). + // OUTPUT: + // gradients to backprop + template + void Operate(OpKernelContext* context, const Tensor& g, const Tensor& a, + Tensor* output) { + OperateNoTemplate(context, g, a, alpha_, output); + } + + private: + T alpha_; +}; + +template +void LeakyReluGradOp::OperateNoTemplate(OpKernelContext* context, + const Tensor& g, const Tensor& a, T alpha, Tensor* output) { + if (!ReluHelpers::ValidateSameSize(context, g, a)) return; + functor::LeakyReluGrad functor; + functor(context->eigen_device(), g.flat(), a.flat(), alpha, + output->flat()); +}; + template class EluOp : public UnaryElementWiseOp> { public: diff --git a/tensorflow/core/kernels/relu_op_functor.h b/tensorflow/core/kernels/relu_op_functor.h index 3bc5ba8a50..7f0951451d 100644 --- a/tensorflow/core/kernels/relu_op_functor.h +++ b/tensorflow/core/kernels/relu_op_functor.h @@ -91,6 +91,37 @@ struct Relu6Grad { } }; + +// Functor used by LeakyReluOp to do the computations. +template +struct LeakyRelu { + // Computes LeakyRelu activation. + // + // features: any shape. + // activations: same shape as "features". + void operator()(const Device& d, typename TTypes::ConstTensor features, + T alpha, typename TTypes::Tensor activations) { + activations.device(d) = features.cwiseMax(features * alpha); + } +}; + +// Functor used by LeakyReluGradOp to do the computations. +template +struct LeakyReluGrad { + // Computes LeakyReluGrad backprops. + // + // gradients: gradients backpropagated to the LeakyRelu op. + // features: either the inputs that were passed to the LeakyRelu or, or its + // outputs (using either one yields the same result here). + // backprops: gradients to backpropagate to the LeakyRelu inputs. + void operator()(const Device& d, typename TTypes::ConstTensor gradients, + typename TTypes::ConstTensor features, T alpha, + typename TTypes::Tensor backprops) { + backprops.device(d) = + (features > static_cast(0)).select(gradients, gradients * alpha); + } +}; + // Functor used by EluOp to do the computations. template struct Elu { diff --git a/tensorflow/core/kernels/relu_op_gpu.cu.cc b/tensorflow/core/kernels/relu_op_gpu.cu.cc index 089ca8ed27..4452f4dcc9 100644 --- a/tensorflow/core/kernels/relu_op_gpu.cu.cc +++ b/tensorflow/core/kernels/relu_op_gpu.cu.cc @@ -114,14 +114,16 @@ struct ReluGrad { } // namespace functor // Definition of the GPU implementations declared in relu_op.cc. -#define DEFINE_GPU_KERNELS(T) \ - template struct functor::Relu; \ - template struct functor::ReluGrad; \ - template struct functor::Relu6; \ - template struct functor::Relu6Grad; \ - template struct functor::Elu; \ - template struct functor::EluGrad; \ - template struct functor::Selu; \ +#define DEFINE_GPU_KERNELS(T) \ + template struct functor::Relu; \ + template struct functor::ReluGrad; \ + template struct functor::Relu6; \ + template struct functor::Relu6Grad; \ + template struct functor::LeakyRelu; \ + template struct functor::LeakyReluGrad; \ + template struct functor::Elu; \ + template struct functor::EluGrad; \ + template struct functor::Selu; \ template struct functor::SeluGrad; TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS); diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index e0f25fb4ef..023f988f80 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -983,6 +983,21 @@ REGISTER_OP("Relu6Grad") .Attr("T: realnumbertype") .SetShapeFn(shape_inference::MergeBothInputsShapeFn); +REGISTER_OP("LeakyRelu") + .Input("features: T") + .Output("activations: T") + .Attr("alpha: float = 0.2") + .Attr("T: {half, float, double} = DT_FLOAT") + .SetShapeFn(shape_inference::UnchangedShape); + +REGISTER_OP("LeakyReluGrad") + .Input("gradients: T") + .Input("features: T") + .Output("backprops: T") + .Attr("alpha: float = 0.2") + .Attr("T: {half, float, double} = DT_FLOAT") + .SetShapeFn(shape_inference::MergeBothInputsShapeFn); + REGISTER_OP("Elu") .Input("features: T") .Output("activations: T") diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index f2595279e0..837e91bc23 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -13604,6 +13604,74 @@ op { minimum: 1 } } +op { + name: "LeakyRelu" + input_arg { + name: "features" + type_attr: "T" + } + output_arg { + name: "activations" + type_attr: "T" + } + attr { + name: "alpha" + type: "float" + default_value { + f: 0.2 + } + } + attr { + name: "T" + type: "type" + default_value { + type: DT_FLOAT + } + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } +} +op { + name: "LeakykReluGrad" + input_arg { + name: "gradients" + type_attr: "T" + } + input_arg { + name: "features" + type_attr: "T" + } + output_arg { + name: "backprops" + type_attr: "T" + } + attr { + name: "alpha" + type: "float" + default_value { + f: 0.2 + } + } + attr { + name: "T" + type: "type" + default_value { + type: DT_FLOAT + } + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } +} op { name: "LearnedUnigramCandidateSampler" input_arg { diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 2d54555cd3..9b3b5fd7aa 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1730,6 +1730,7 @@ bool OpDoesntRequireOutput(const string& op_name) { "SoftplusGrad", "Softsign", "ReluGrad", + "LeakyReluGrad", "Conv2D", "DepthwiseConv2dNative", "Dilation2D", @@ -1799,6 +1800,7 @@ bool OpDoesntRequireInput(const string& op_name) { "BiasAdd", "Relu", "Relu6", + "LeakyRelu", "Elu", "Selu", "SparseSoftmaxCrossEntropyWithLogits", diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index 25e947f09e..ccb3a231bb 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -252,6 +252,119 @@ class Relu6Test(test.TestCase): self.assertLess(err, 1e-10) +class LeakyReluTest(test.TestCase): + + def _npLeakyRelu(self, np_features, alpha=0.1): + return np.maximum(np_features, alpha * np_features) + + def testNpLeakyRelu(self): + self.assertAllClose( + np.array([[-0.09, 0.7, -0.05, 0.3, -0.01], + [0.1, -0.03, 0.5, -0.07, 0.9]]), + self._npLeakyRelu( + np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 0.9] + ]), alpha=0.1)) + + def _testLeakyRelu(self, np_features, alpha, use_gpu=False): + np_leaky_relu = self._npLeakyRelu(np_features, alpha) + with self.test_session(use_gpu=use_gpu): + leaky_relu = nn_ops.leaky_relu(np_features, alpha) + tf_leaky_relu = leaky_relu.eval() + self.assertAllClose(np_leaky_relu, tf_leaky_relu) + self.assertShapeEqual(np_leaky_relu, leaky_relu) + + def testNumbers(self): + for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: + self._testLeakyRelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), + alpha=0.2, use_gpu=False) + if t in [np.float16, np.float32, np.float64]: + self._testLeakyRelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), + alpha=0.1, use_gpu=True) + + # The gradient test for ReLU is a bit tricky as the derivative is not well + # defined at around zero and we want to avoid that in terms of input values. + def testGradientFloat32(self): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + name="x") + y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float32, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], y, [2, 5], x_init_value=x_init) + print("leaky_relu (float32) gradient err = ", err) + self.assertLess(err, 1e-4) + + def testGradientFloat64(self): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + dtype=dtypes.float64, + name="x") + y = nn_ops.leaky_relu(x, alpha=0.2, name="leaky_relu") + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float64, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], y, [2, 5], x_init_value=x_init) + print("leaky_relu (float64) gradient err = ", err) + self.assertLess(err, 1e-10) + + def testGradGradFloat32(self): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + name="x") + y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float32, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) + print("leaky_relu (float32) gradient of gradient err = ", err) + self.assertLess(err, 1e-4) + + def testGradGradFloat64(self): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + dtype=dtypes.float64, + name="x") + y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float64, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) + print("leaky_relu (float64) gradient of gradient err = ", err) + self.assertLess(err, 1e-10) + + def testGradientScalar(self): + with self.test_session() as sess: + x = variables.Variable(-100.) + y = nn_ops.leaky_relu(x, 0.05) + loss = y**2 + optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) + train_op = optimizer.minimize(loss) + sess.run(variables.global_variables_initializer()) + sess.run(train_op) + self.assertAllClose(x.eval(), -99.9) + + class EluTest(test.TestCase): def _npElu(self, np_features): diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index df23ac55ce..c2dd58bdf0 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -390,6 +390,21 @@ def _Relu6GradGrad(op, grad): array_ops.zeros(shape=array_ops.shape(x), dtype=x.dtype)) +@ops.RegisterGradient("LeakyRelu") +def _LeakyReluGrad(op, grad): + x = op.inputs[0] + alpha = op.get_attr("alpha") + return gen_nn_ops.leaky_relu_grad(grad, x, alpha=alpha) + + +@ops.RegisterGradient("LeakyReluGrad") +def _LeakyReluGradGrad(op, grad): + x = op.inputs[1] + alpha = op.get_attr("alpha") + return (gen_nn_ops.leaky_relu_grad(grad, x, alpha=alpha), + array_ops.zeros(shape=array_ops.shape(x), dtype=x.dtype)) + + @ops.RegisterGradient("Elu") def _EluGrad(op, grad): return gen_nn_ops.elu_grad(grad, op.outputs[0]) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 6fd1273687..31b8f3945d 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1601,8 +1601,7 @@ def leaky_relu(features, alpha=0.2, name=None): features = ops.convert_to_tensor(features, name="features") if features.dtype.is_integer: features = math_ops.to_float(features) - alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha") - return math_ops.maximum(alpha * features, features, name=name) + return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name) def _flatten_outer_dims(logits): -- GitLab From bc5d68b95a8ca1410905d532d1c356dd5c76cd30 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 21 Aug 2018 20:56:48 +0000 Subject: [PATCH 0022/1085] Update test case to use file_io.get_matching_files Signed-off-by: Yong Tang --- tensorflow/python/lib/io/file_io_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py index 1d247aa8ba..d17c02fe9e 100644 --- a/tensorflow/python/lib/io/file_io_test.py +++ b/tensorflow/python/lib/io/file_io_test.py @@ -23,7 +23,6 @@ import os.path from tensorflow.python.framework import errors from tensorflow.python.lib.io import file_io -from tensorflow.python.ops import gen_io_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -585,7 +584,6 @@ class FileIoTest(test.TestCase): self.assertEqual(crc2, crc3) def testMatchingFilesPermission(self): - # Test case for GitHub issue 19274. # Create top level directory test_dir. dir_path = os.path.join(self._base_dir, "test_dir") file_io.create_dir(dir_path) @@ -602,10 +600,11 @@ class FileIoTest(test.TestCase): file_io.FileIO(file_path, mode="w").write("testing") # Change noread to noread access. os.chmod(noread_path, 0) - expected_match = [compat.as_bytes(dir_path)] - with self.test_session() as sess: - self.assertItemsEqual( - gen_io_ops.matching_files(dir_path).eval(), expected_match) + expected_match = [ + compat.as_bytes(os.path.join(any_path, name)) for name in files] + self.assertItemsEqual( + file_io.get_matching_files(os.path.join(dir_path, "*", "file*.txt")), + expected_match) # Change noread back so that it could be cleaned during tearDown. os.chmod(noread_path, 0o777) -- GitLab From f7a1522354be98af329f3a1b4047a7b3429bce9c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 21 Aug 2018 23:53:55 +0000 Subject: [PATCH 0023/1085] Fix python 3 issues casued by compat.as_bytes Signed-off-by: Yong Tang --- tensorflow/python/lib/io/file_io_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py index d17c02fe9e..d130fae34e 100644 --- a/tensorflow/python/lib/io/file_io_test.py +++ b/tensorflow/python/lib/io/file_io_test.py @@ -601,7 +601,7 @@ class FileIoTest(test.TestCase): # Change noread to noread access. os.chmod(noread_path, 0) expected_match = [ - compat.as_bytes(os.path.join(any_path, name)) for name in files] + os.path.join(any_path, name) for name in files] self.assertItemsEqual( file_io.get_matching_files(os.path.join(dir_path, "*", "file*.txt")), expected_match) -- GitLab From f4df6cb3aebc64a8e9c2c2d2dc06fa039e188566 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 22 Aug 2018 13:24:05 +0000 Subject: [PATCH 0024/1085] Remove unused compat import Signed-off-by: Yong Tang --- tensorflow/python/lib/io/file_io_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py index d130fae34e..33cea08045 100644 --- a/tensorflow/python/lib/io/file_io_test.py +++ b/tensorflow/python/lib/io/file_io_test.py @@ -24,7 +24,6 @@ import os.path from tensorflow.python.framework import errors from tensorflow.python.lib.io import file_io from tensorflow.python.platform import test -from tensorflow.python.util import compat class FileIoTest(test.TestCase): -- GitLab From cb5c61a3e11a37fb39a246aaf8ed6d02dd9ae9ab Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Fri, 24 Aug 2018 11:51:34 +0800 Subject: [PATCH 0025/1085] Refine LeakyRelu codes and update APIs. --- .../api_def/base_api/api_def_LeakyRelu.pbtxt | 4 ++++ .../base_api/api_def_LeakyReluGrad.pbtxt | 24 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 2 +- tensorflow/python/eager/pywrap_tfe_src.cc | 2 +- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt new file mode 100644 index 0000000000..4a61889f54 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "LeakyRelu" + summary: "Computes rectified linear: `max(features, features * alpha)`." +} diff --git a/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt new file mode 100644 index 0000000000..e427526602 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt @@ -0,0 +1,24 @@ +op { + graph_op_name: "LeakyReluGrad" + visibility: HIDDEN + in_arg { + name: "gradients" + description: < 0) + alpha * gradients * (featurs <= 0)`. +END + } + summary: "Computes rectified linear gradients for a LeakyRelu operation." +} diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 837e91bc23..7693c2d485 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -13637,7 +13637,7 @@ op { } } op { - name: "LeakykReluGrad" + name: "LeakyReluGrad" input_arg { name: "gradients" type_attr: "T" diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 9b3b5fd7aa..18fafd0de1 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1730,6 +1730,7 @@ bool OpDoesntRequireOutput(const string& op_name) { "SoftplusGrad", "Softsign", "ReluGrad", + "LeakyRelu", "LeakyReluGrad", "Conv2D", "DepthwiseConv2dNative", @@ -1800,7 +1801,6 @@ bool OpDoesntRequireInput(const string& op_name) { "BiasAdd", "Relu", "Relu6", - "LeakyRelu", "Elu", "Selu", "SparseSoftmaxCrossEntropyWithLogits", -- GitLab From 877358f68fcfd3ca06fdec87007e0cc90502f202 Mon Sep 17 00:00:00 2001 From: David Norman Date: Fri, 24 Aug 2018 17:13:53 +0100 Subject: [PATCH 0026/1085] Allow for disabling these tests via manifest --- tensorflow/compiler/xla/tests/while_test.cc | 46 ++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc index 1bdf1867b9..e6c69a5a86 100644 --- a/tensorflow/compiler/xla/tests/while_test.cc +++ b/tensorflow/compiler/xla/tests/while_test.cc @@ -48,7 +48,7 @@ class WhileTest : public ClientLibraryTestBase {}; // while (result < 5) { // result = result + 1; // } -TEST_F(WhileTest, WhileWithScalarS32Result) { +XLA_TEST_F(WhileTest, WhileWithScalarS32Result) { auto result_shape = ShapeUtil::MakeShape(S32, {}); // Create a computation for the condition: repeat for 5 iterations. @@ -84,7 +84,7 @@ TEST_F(WhileTest, WhileWithScalarS32Result) { // while (result < 5) { // result = result + 1; // } -TEST_F(WhileTest, WhileWithScalarS64Result) { +XLA_TEST_F(WhileTest, WhileWithScalarS64Result) { auto result_shape = ShapeUtil::MakeShape(S64, {}); // Create a computation for the condition: repeat for 5 iterations. @@ -114,7 +114,7 @@ TEST_F(WhileTest, WhileWithScalarS64Result) { ComputeAndCompareR0(&builder, 5, {}); } -TEST_F(WhileTest, WhileWithScalarResultNonConstInit) { +XLA_TEST_F(WhileTest, WhileWithScalarResultNonConstInit) { auto result_shape = ShapeUtil::MakeShape(S32, {}); auto orig_shape = ShapeUtil::MakeShape(S32, {2}); @@ -147,7 +147,7 @@ TEST_F(WhileTest, WhileWithScalarResultNonConstInit) { ComputeAndCompareR0(&builder, 5, {}); } -TEST_F(WhileTest, WhileWithPredicateResult) { +XLA_TEST_F(WhileTest, WhileWithPredicateResult) { auto result_shape = ShapeUtil::MakeShape(PRED, {}); // Create a computation for the condition: run until condition is true. @@ -184,7 +184,7 @@ TEST_F(WhileTest, WhileWithPredicateResult) { // while (result.sum() < 15.5f) { // result = result + vector(0); // } -TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithEmptyVectorResult)) { +XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithEmptyVectorResult)) { Shape result_shape = ShapeUtil::MakeShape(F32, {0}); // Create a computation for the reduction. @@ -238,7 +238,7 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithEmptyVectorResult)) { // while (result.sum() < 15.5f) { // result = result + vector(8, 0.125f); // } -TEST_F(WhileTest, WhileWithVectorResult) { +XLA_TEST_F(WhileTest, WhileWithVectorResult) { Shape result_shape = ShapeUtil::MakeShape(F32, {8}); // Create a computation for the reduction. @@ -298,7 +298,7 @@ TEST_F(WhileTest, WhileWithVectorResult) { // result = result + vector(8, 0.125f); // } // tuple = tuple { while } -TEST_F(WhileTest, WhileWithVectorResultIntoTuple) { +XLA_TEST_F(WhileTest, WhileWithVectorResultIntoTuple) { Shape result_shape = ShapeUtil::MakeShape(F32, {8}); // Create a computation for the reduction. @@ -353,7 +353,7 @@ TEST_F(WhileTest, WhileWithVectorResultIntoTuple) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0.0001)); } -TEST_F(WhileTest, WhileWithPermutationAndTupleResult) { +XLA_TEST_F(WhileTest, WhileWithPermutationAndTupleResult) { std::vector shape_elements = { ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3})}; @@ -408,7 +408,7 @@ TEST_F(WhileTest, WhileWithPermutationAndTupleResult) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0.0001)); } -TEST_F(WhileTest, WhileWithPermutationAndVectorResult) { +XLA_TEST_F(WhileTest, WhileWithPermutationAndVectorResult) { std::vector shape_elements = { ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3})}; @@ -466,7 +466,7 @@ TEST_F(WhileTest, WhileWithPermutationAndVectorResult) { // get<0>(result) = get<0>(result) + 1; // get<1>(result) = get<1>(result) + vector(10, 1.0f); // } -TEST_F(WhileTest, WhileWithTupleResult) { +XLA_TEST_F(WhileTest, WhileWithTupleResult) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {10})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -516,7 +516,7 @@ TEST_F(WhileTest, WhileWithTupleResult) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0.0001)); } -TEST_F(WhileTest, WhileWithPredicateTupleResult) { +XLA_TEST_F(WhileTest, WhileWithPredicateTupleResult) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(PRED, {})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -562,7 +562,7 @@ TEST_F(WhileTest, WhileWithPredicateTupleResult) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0)); } -TEST_F(WhileTest, WhileWithTupleConstantScalarResult) { +XLA_TEST_F(WhileTest, WhileWithTupleConstantScalarResult) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(S32, {})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -622,7 +622,7 @@ TEST_F(WhileTest, WhileWithTupleConstantScalarResult) { // get<1>(w1) = get<1>(w1) + vector(10, 1.0f); // } // result = get<1>(w0) + get<1>(w1) -TEST_F(WhileTest, TwoWhileWithTupleResult) { +XLA_TEST_F(WhileTest, TwoWhileWithTupleResult) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {10})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -701,7 +701,7 @@ TEST_F(WhileTest, TwoWhileWithTupleResult) { } // Test while nodes that share the while body computation. -TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) { +XLA_TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {10})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -768,7 +768,7 @@ TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) { // Test while nodes that share the while body computation. // TODO(b/37245345): Fails on GPU backend. -TEST_F(WhileTest, DISABLED_ON_GPU(WhileLoopsWithSharedBodyAndInit)) { +XLA_TEST_F(WhileTest, DISABLED_ON_GPU(WhileLoopsWithSharedBodyAndInit)) { std::vector shape_elements = {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {10})}; Shape result_shape = ShapeUtil::MakeTupleShape(shape_elements); @@ -907,7 +907,7 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) { // Per backend the values generated can be different as the different backends // use different random number generators. // TODO(b/32240857): Extend test to verify outputs. -TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) { +XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) { auto v6s32 = ShapeUtil::MakeShape(S32, {6}); // Create a computation for the condition: repeat for count iterations. @@ -953,7 +953,7 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) { } } -TEST_F(WhileTest, WhileThatSwapsParameterWithTupleElement) { +XLA_TEST_F(WhileTest, WhileThatSwapsParameterWithTupleElement) { auto element_shape = ShapeUtil::MakeShape(F32, {2}); XlaBuilder outer("outer"); @@ -985,7 +985,7 @@ TEST_F(WhileTest, WhileThatSwapsParameterWithTupleElement) { ErrorSpec(1e-6)); } -TEST_F(WhileTest, WhileThatSwapsParameterWithBroadcast) { +XLA_TEST_F(WhileTest, WhileThatSwapsParameterWithBroadcast) { auto element_shape = ShapeUtil::MakeShape(F32, {2}); XlaBuilder outer("outer"); @@ -1010,7 +1010,7 @@ TEST_F(WhileTest, WhileThatSwapsParameterWithBroadcast) { ErrorSpec(1e-6)); } -TEST_F(WhileTest, WhileThatTurnsScalarParameterToTupleElement) { +XLA_TEST_F(WhileTest, WhileThatTurnsScalarParameterToTupleElement) { auto element_shape = ShapeUtil::MakeShape(F32, {}); XlaBuilder outer("outer"); @@ -1044,7 +1044,7 @@ TEST_F(WhileTest, WhileThatTurnsScalarParameterToTupleElement) { // result[0] = result[0] + 1; // result[1] = result[1] + 1; // } -TEST_F(WhileTest, WhileWithMixedTupleElements) { +XLA_TEST_F(WhileTest, WhileWithMixedTupleElements) { auto result_shape = ShapeUtil::MakeTupleShape( {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(S32, {})}); @@ -1152,7 +1152,7 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) { // while (f(result).get<0>()) { // result = result + 1; // } -TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) { +XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) { auto result_shape = ShapeUtil::MakeShape(S32, {}); // Create a computation for the condition: repeat for 5 iterations. @@ -1192,7 +1192,7 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) { ComputeAndCompareR0(&builder, 5, {}); } -TEST_F(WhileTest, WhileWithLoopInvariantOperation) { +XLA_TEST_F(WhileTest, WhileWithLoopInvariantOperation) { auto matrix_shape = ShapeUtil::MakeShape(F32, {2, 2}); auto scalar_s32 = ShapeUtil::MakeShape(S32, {}); auto while_shape = ShapeUtil::MakeTupleShape( @@ -1236,7 +1236,7 @@ TEST_F(WhileTest, WhileWithLoopInvariantOperation) { {param_value.get()}, ErrorSpec(4e-5)); } -TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileInfeedCondition)) { +XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileInfeedCondition)) { auto while_shape = ShapeUtil::MakeShape(S32, {}); XlaComputation condition; -- GitLab From 7a54c15804f7bb0d0c40fea5c84b1f4acee58bac Mon Sep 17 00:00:00 2001 From: Stefan Dyulgerov Date: Sat, 25 Aug 2018 13:18:11 +0300 Subject: [PATCH 0027/1085] upgraded protobuf to v.3.6.1 --- tensorflow/contrib/cmake/external/protobuf.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake index f56fb35a0f..56a57a2340 100644 --- a/tensorflow/contrib/cmake/external/protobuf.cmake +++ b/tensorflow/contrib/cmake/external/protobuf.cmake @@ -16,7 +16,7 @@ include (ExternalProject) set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src) set(PROTOBUF_URL https://github.com/google/protobuf.git) -set(PROTOBUF_TAG v3.6.0) +set(PROTOBUF_TAG v3.6.1) if(WIN32) if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") -- GitLab From e93a9f9ccfd9c7a2419bf3fc1d7866765bbcfce3 Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Tue, 28 Aug 2018 18:55:51 -0700 Subject: [PATCH 0028/1085] Update GPU occupancy checking to utilize CUDA's occupancy calculator functions -Replace references to the UnqueryableDeviceParams struct with calls to CUDA's built-in occupancy calculation functions -Update calls to the occupancy checking functions with the new changes -Changes should provide more long-term reliability and will remove the need to manually update hardcoded data values for new GPU architectures --- .../xla/service/gpu/partition_assignment.cc | 9 +- .../stream_executor/cuda/cuda_gpu_executor.cc | 192 ++---------------- .../stream_executor/device_description.cc | 98 +++------ .../stream_executor/device_description.h | 73 ++----- 4 files changed, 61 insertions(+), 311 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/partition_assignment.cc b/tensorflow/compiler/xla/service/gpu/partition_assignment.cc index cf9f102d31..375f68a159 100644 --- a/tensorflow/compiler/xla/service/gpu/partition_assignment.cc +++ b/tensorflow/compiler/xla/service/gpu/partition_assignment.cc @@ -62,13 +62,8 @@ LaunchDimensions CalculateLaunchDimensions( // // * = - auto threads_per_core = device_desc.threads_per_core_limit(); - auto blocks_per_core = device_desc.blocks_per_core_limit(); - int64 threads_per_block; - if (threads_per_core != 0 && blocks_per_core != 0) { - threads_per_block = device_desc.threads_per_core_limit() / - device_desc.blocks_per_core_limit(); - } else { + int64 threads_per_block = device_desc.threads_per_block_limit(); + if (threads_per_block == 0) { static std::atomic log_count{0}; if (log_count.fetch_add(1) < 8) { LOG(WARNING) << "Attempting to calculate launch dimensions for GPU " diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index e30f50ea2a..39b0696c93 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -467,33 +467,26 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, return; } + int block_size = thread_dims.x * thread_dims.y * thread_dims.z; + const DeviceDescription &device_description = kernel.parent()->GetDeviceDescription(); - uint64 blocks_per_sm = CalculateOccupancy( - device_description, regs_per_thread, smem_per_block, thread_dims); - VLOG(2) << "Resident blocks per SM is " << blocks_per_sm; + const CUDAKernel* cuda_kernel = AsCUDAKernel(&kernel); + CUfunction cufunc = cuda_kernel->AsCUDAFunctionValue(); - // To increase occupancy, there must be a sufficient number of blocks - // available to spread across the sm's at this new improved occupancy level. - int multiprocessor_count = device_description.core_count(); - int block_count = block_dims.x * block_dims.y * block_dims.z; - int available_blocks_per_sm = - port::MathUtil::CeilOfRatio(block_count, multiprocessor_count); - if (available_blocks_per_sm <= static_cast(blocks_per_sm)) { - VLOG(2) << "Occupancy is limited by number of blocks available per sm."; - return; - } + int blocks_per_sm = CalculateOccupancy(device_description, regs_per_thread, + smem_per_block, thread_dims, cufunc); + VLOG(2) << "Resident blocks per SM is " << blocks_per_sm; - uint64 improved_regs_per_thread = CalculateRegisterLimitForTargetOccupancy( - device_description, smem_per_block, thread_dims, blocks_per_sm + 1); - if (improved_regs_per_thread != 0) { - VLOG(2) << "Reducing register usage from " << regs_per_thread - << " to " << improved_regs_per_thread - << " could increase resident blocks per SM by one."; - } else { - VLOG(2) << "Resident blocks per SM cannot be increased by reducing " - "register usage."; + int suggested_threads = + CompareOccupancy(&blocks_per_sm, device_description, regs_per_thread, + smem_per_block, thread_dims, cufunc); + if (suggested_threads != 0) { + VLOG(2) << "The cuda occupancy calculator reccommends using " + << suggested_threads + << " threads per block to acheive an occupancy of " << blocks_per_sm + << " blocks per SM."; } } @@ -980,144 +973,6 @@ static int TryToReadNumaNode(const string &pci_bus_id, int device_ordinal) { #endif } -// Set of compute capability specific device parameters that cannot be -// queried from the driver API. These values instead are baked into a -// lookup table indexed by compute capability version. -struct UnqueryableDeviceParams { - int cc_major; - int cc_minor; - uint64 blocks_per_core_limit; - uint64 registers_per_core_limit; - uint64 registers_per_thread_limit; - uint64 warp_alloc_granularity; - uint64 register_alloc_granularity; - uint64 shared_memory_alloc_granularity; -}; - -// http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#compute-capabilities -// https://developer.download.nvidia.com/compute/cuda/CUDA_Occupancy_calculator.xls -static const UnqueryableDeviceParams kAllUnqueryableDeviceParams[] = { - { - 2, 0, // compute capability (2.0) - 8, // blocks_per_core_limit - 32 * 1024, // registers_per_core_limit - 63, // registers_per_thread_limit - 2, // warp_alloc_granularity - 64, // register_alloc_granularity - 128, // shared_memory_alloc_granularity - }, - { - 2, 1, // compute capability (2.1) - 8, // blocks_per_core_limit - 32 * 1024, // registers_per_core_limit - 63, // registers_per_thread_limit - 2, // warp_alloc_granularity - 64, // register_alloc_granularity - 128, // shared_memory_alloc_granularity - }, - { - 3, 0, // compute capability (3.0) - 16, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 63, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 3, 2, // compute capability (3.2) - 16, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 3, 5, // compute capability (3.5) - 16, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 3, 7, // compute capability (3.7) - 16, // blocks_per_core_limit - 128 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 5, 0, // compute capability (5.0) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 5, 2, // compute capability (5.2) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 5, 3, // compute capability (5.3) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 6, 0, // compute capability (6.0) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 2, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 6, 1, // compute capability (6.1) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - { - 6, 2, // compute capability (6.2) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 4, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, - // TODO(jlebar): Confirm the alloc granularity values for sm_70. These are - // not published in the spreadsheet linked above. Currently we guess that - // they're the same as sm_60. - { - 7, 0, // compute capability (7.0) - 32, // blocks_per_core_limit - 64 * 1024, // registers_per_core_limit - 255, // registers_per_thread_limit - 2, // warp_alloc_granularity - 256, // register_alloc_granularity - 256, // shared_memory_alloc_granularity - }, -}; DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const { internal::DeviceDescriptionBuilder builder; @@ -1193,19 +1048,6 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const { builder.set_name(device_name); } - for (size_t i = 0; i < TF_ARRAYSIZE(kAllUnqueryableDeviceParams); i++) { - const auto ¶ms = kAllUnqueryableDeviceParams[i]; - if (params.cc_major == cc_major_ && params.cc_minor == cc_minor_) { - builder.set_blocks_per_core_limit(params.blocks_per_core_limit); - builder.set_registers_per_core_limit(params.registers_per_core_limit); - builder.set_registers_per_thread_limit(params.registers_per_thread_limit); - builder.set_warp_alloc_granularity(params.warp_alloc_granularity); - builder.set_register_alloc_granularity(params.register_alloc_granularity); - builder.set_shared_memory_alloc_granularity( - params.shared_memory_alloc_granularity); - } - } - builder.set_platform_version( port::StrCat("Compute Capability ", cc_major_, ".", cc_minor_)); @@ -1227,6 +1069,10 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const { CUDADriver::GetMaxRegistersPerBlock(device_).ValueOrDie()); builder.set_threads_per_warp( CUDADriver::GetThreadsPerWarp(device_).ValueOrDie()); + builder.set_registers_per_core_limit( + CUDADriver::GetDeviceAttribute( + CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR, device_) + .ValueOrDie()); auto built = builder.Build(); return built.release(); diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc index 8ca0677f8a..df52ce6cce 100644 --- a/tensorflow/stream_executor/device_description.cc +++ b/tensorflow/stream_executor/device_description.cc @@ -37,16 +37,11 @@ DeviceDescription::DeviceDescription() kUninitializedUint64), block_dim_limit_(kUninitializedUint64, kUninitializedUint64, kUninitializedUint64), - blocks_per_core_limit_(kUninitializedUint64), threads_per_core_limit_(kUninitializedUint64), threads_per_block_limit_(kUninitializedUint64), threads_per_warp_(kUninitializedUint64), registers_per_core_limit_(kUninitializedUint64), registers_per_block_limit_(kUninitializedUint64), - registers_per_thread_limit_(kUninitializedUint64), - warp_alloc_granularity_(1), - register_alloc_granularity_(1), - shared_memory_alloc_granularity_(1), device_address_bits_(kUninitializedUint64), device_memory_size_(kUninitializedUint64), memory_bandwidth_(kUninitializedUint64), @@ -162,75 +157,36 @@ static uint64 RoundDown(uint64 value, uint64 n) { return port::MathUtil::FloorOfRatio(value, n) * n; } -uint64 CalculateOccupancy(const DeviceDescription &device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim &thread_dims) { - // Don't try to compute occupancy if necessary values are not initialized. - uint64 required_fields[] = { device_description.registers_per_thread_limit(), - device_description.threads_per_warp(), - device_description.warp_alloc_granularity(), - device_description.register_alloc_granularity(), - device_description.registers_per_block_limit(), - device_description.shared_memory_per_core(), - device_description.blocks_per_core_limit() }; - for (auto value : required_fields) { - if (value == kUninitializedUint64) { - return 0; - } - } - - if (registers_per_thread > device_description.registers_per_thread_limit()) { - return 0; - } - - uint64 warps_per_block = - port::MathUtil::CeilOfRatio(thread_dims.x * thread_dims.y * thread_dims.z, - device_description.threads_per_warp()); - - // Warp resources are allocated at a particular granularity. This value is - // the effective number of warps for resource allocation purposes. - uint64 alloc_warps_per_block = - RoundUp(warps_per_block, device_description.warp_alloc_granularity()); - - uint64 alloc_regs_per_warp = - RoundUp(device_description.threads_per_warp() * registers_per_thread, - device_description.register_alloc_granularity()); - uint64 regs_per_block = alloc_warps_per_block * alloc_regs_per_warp; - uint64 reg_limit = - device_description.registers_per_block_limit() / regs_per_block; - - uint64 alloc_smem_per_block = RoundUp( - shared_memory_per_block, - device_description.shared_memory_alloc_granularity()); - uint64 smem_limit = alloc_smem_per_block > 0 ? - device_description.shared_memory_per_core() / alloc_smem_per_block : - device_description.blocks_per_core_limit(); - - uint64 thread_limit = device_description.threads_per_core_limit() - / (warps_per_block * device_description.threads_per_warp()); - - return std::min({ device_description.blocks_per_core_limit(), - reg_limit, smem_limit, thread_limit }); +int CalculateOccupancy(const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func) { + int suggested_blocks = 0; + int suggested_threads = 0; + CUresult err = + cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, + func, NULL, shared_memory_per_block, 0); + CHECK_EQ(err, CUDA_SUCCESS); + return suggested_blocks; } -uint64 CalculateRegisterLimitForTargetOccupancy( - const DeviceDescription &device_description, uint64 shared_memory_per_block, - const ThreadDim &thread_dims, uint64 target_blocks_per_core) { - // Linear search from maximum number of registers down until the target - // blocks per SM is found. - // TODO(meheff): Compute this using a closed form solution. - int reg_step = device_description.register_alloc_granularity() / - device_description.threads_per_warp(); - for (int r = device_description.registers_per_thread_limit(); r > 0; - r = RoundDown(r - 1, reg_step)) { - uint64 occupancy = CalculateOccupancy( - device_description, r, shared_memory_per_block, thread_dims); - if (occupancy >= target_blocks_per_core) { - return r; - } +int CompareOccupancy(int* initial_blocks, + const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func) { + int suggested_blocks = 0; + int suggested_threads = 0; + CUresult err = + cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, + func, NULL, shared_memory_per_block, 0); + CHECK_EQ(err, CUDA_SUCCESS); + if (suggested_blocks > *initial_blocks) { + *initial_blocks = suggested_blocks; + return suggested_threads; + } else { + return 0; } - return 0; } } // namespace stream_executor diff --git a/tensorflow/stream_executor/device_description.h b/tensorflow/stream_executor/device_description.h index 7f99d81ef3..d335b9b875 100644 --- a/tensorflow/stream_executor/device_description.h +++ b/tensorflow/stream_executor/device_description.h @@ -24,6 +24,7 @@ limitations under the License. #include #include "tensorflow/stream_executor/platform/port.h" +#include "tensorflow/stream_executor/cuda/cuda_driver.h" #include "tensorflow/stream_executor/launch_dim.h" #include "tensorflow/stream_executor/platform/port.h" @@ -79,10 +80,6 @@ class DeviceDescription { // legitimate kernel launch request. const BlockDim &block_dim_limit() const { return block_dim_limit_; } - // Returns the limit on the number of simultaneously resident blocks - // on a multiprocessor. - uint64 blocks_per_core_limit() const { return blocks_per_core_limit_; } - // Returns the limit on the total number of threads that can be launched in a // single block; i.e. the limit on x * y * z dimensions of a ThreadDim. // This limit affects what constitutes a legitimate kernel launch request. @@ -110,27 +107,6 @@ class DeviceDescription { return registers_per_block_limit_; } - // Returns the limit on the total number of registers that can be - // allocated to a thread. - const uint64 ®isters_per_thread_limit() const { - return registers_per_thread_limit_; - } - - // Returns the granularity at which warps are allocated resources. - const uint64 &warp_alloc_granularity() const { - return warp_alloc_granularity_; - } - - // Returns the granularity at which registers are allocated to warps. - const uint64 ®ister_alloc_granularity() const { - return register_alloc_granularity_; - } - - // Returns the granularity at which shared memory is allocated to warps. - const uint64 &shared_memory_alloc_granularity() const { - return shared_memory_alloc_granularity_; - } - // Returns the number of address bits available to kernel code running on the // platform. This affects things like the maximum allocation size and perhaps // types used in kernel code such as size_t. @@ -200,19 +176,12 @@ class DeviceDescription { ThreadDim thread_dim_limit_; BlockDim block_dim_limit_; - uint64 blocks_per_core_limit_; - uint64 threads_per_core_limit_; uint64 threads_per_block_limit_; uint64 threads_per_warp_; uint64 registers_per_core_limit_; uint64 registers_per_block_limit_; - uint64 registers_per_thread_limit_; - - uint64 warp_alloc_granularity_; - uint64 register_alloc_granularity_; - uint64 shared_memory_alloc_granularity_; uint64 device_address_bits_; uint64 device_memory_size_; @@ -270,10 +239,6 @@ class DeviceDescriptionBuilder { device_description_->block_dim_limit_ = value; } - void set_blocks_per_core_limit(uint64 value) { - device_description_->blocks_per_core_limit_ = value; - } - void set_threads_per_core_limit(uint64 value) { device_description_->threads_per_core_limit_ = value; } @@ -290,19 +255,6 @@ class DeviceDescriptionBuilder { void set_registers_per_block_limit(uint64 value) { device_description_->registers_per_block_limit_ = value; } - void set_registers_per_thread_limit(uint64 value) { - device_description_->registers_per_thread_limit_ = value; - } - - void set_warp_alloc_granularity(uint64 value) { - device_description_->warp_alloc_granularity_ = value; - } - void set_register_alloc_granularity(uint64 value) { - device_description_->register_alloc_granularity_ = value; - } - void set_shared_memory_alloc_granularity(uint64 value) { - device_description_->shared_memory_alloc_granularity_ = value; - } void set_device_address_bits(uint64 value) { device_description_->device_address_bits_ = value; @@ -375,17 +327,18 @@ void CalculateDimensionality(const DeviceDescription &device_description, // Compute and return maximum blocks per core (occupancy) based on the // device description, some kernel characteristics and the number of threads per // block. If unable to compute occupancy, zero is returned. -uint64 CalculateOccupancy(const DeviceDescription &device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim &thread_dims); - -// Compute and return the maximum number of registers per thread which -// achieves the target occupancy. If the target is not possible then -// zero is returned. -uint64 CalculateRegisterLimitForTargetOccupancy( - const DeviceDescription &device_description, uint64 shared_memory_per_block, - const ThreadDim &thread_dims, uint64 target_blocks_per_core); +int CalculateOccupancy(const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func); + +// Compute and return the suggested thread count to acheive ideal occupancy. +// If the provided thread dimensions match this number, zero is returned. +int CompareOccupancy(int* initial_blocks, + const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func); } // namespace stream_executor -- GitLab From 4e72dd865a3fc83baa69f6b7c08720a1b546a464 Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Wed, 29 Aug 2018 17:05:43 +0800 Subject: [PATCH 0029/1085] Refine LeakyRelu codes. 1. Add C++ gradient of gradient definition of LeakyReLu and revalant UT. 2. Using forward compatibility layer for python code changes. --- tensorflow/cc/gradients/nn_grad.cc | 18 ++++- tensorflow/cc/gradients/nn_grad_test.cc | 16 +++++ .../python/kernel_tests/relu_op_test.py | 70 ++++++++++--------- tensorflow/python/ops/nn_ops.py | 5 +- 4 files changed, 73 insertions(+), 36 deletions(-) diff --git a/tensorflow/cc/gradients/nn_grad.cc b/tensorflow/cc/gradients/nn_grad.cc index 0fc23d0bf7..2a32a2ed6f 100644 --- a/tensorflow/cc/gradients/nn_grad.cc +++ b/tensorflow/cc/gradients/nn_grad.cc @@ -149,13 +149,27 @@ Status LeakyReluGradHelper(const Scope& scope, const Operation& op, float alpha; TF_RETURN_IF_ERROR(GetNodeAttr(op.node()->attrs(), "alpha", &alpha)); internal::LeakyReluGrad::Attrs attrs; - attrs.Alpha(alpha); - auto dx = internal::LeakyReluGrad(scope, grad_inputs[0], op.input(0), attrs); + auto dx = internal::LeakyReluGrad(scope, grad_inputs[0], op.input(0), + attrs.Alpha(alpha)); grad_outputs->push_back(dx); return scope.status(); } REGISTER_GRADIENT_OP("LeakyRelu", LeakyReluGradHelper); +Status LeakyReluGradGradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + float alpha; + TF_RETURN_IF_ERROR(GetNodeAttr(op.node()->attrs(), "alpha", &alpha)); + internal::LeakyReluGrad::Attrs attrs; + auto dx = internal::LeakyReluGrad(scope, grad_inputs[0], op.input(1), + attrs.Alpha(alpha)); + grad_outputs->push_back(dx); + grad_outputs->push_back(NoGradient()); + return scope.status(); +} +REGISTER_GRADIENT_OP("LeakyReluGrad", LeakyReluGradGradHelper); + Status EluGradHelper(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index 5ebece7b6e..bf0db1f59d 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/cc/framework/gradient_checker.h" #include "tensorflow/cc/framework/testutil.h" #include "tensorflow/cc/gradients/grad_testutil.h" +#include "tensorflow/cc/ops/nn_ops_internal.h" #include "tensorflow/cc/ops/standard_ops.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status_test_util.h" @@ -173,6 +174,21 @@ TEST_F(NNGradTest, LeakyReluGrad) { RunTest(x, x_init_value, y, shape); } +TEST_F(NNGradTest, LeakyReluGradGrad) { + TensorShape shape({5, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + // Avoid input values where Leaky ReLU gradient is not well defined (around + // zero). + Tensor x_init_value = test::AsTensor( + {2.3f, 1.9f, 1.5f, 1.1f, 0.7f, 0.3f, -0.1f, -0.5f, -0.9f, -1.3f}, + {5, 2}); + Tensor features = test::AsTensor( + {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, + {5, 2}); + auto y = ops::internal::LeakyReluGrad(scope_, x, features); + RunTest(x, x_init_value, y, shape); +} + TEST_F(NNGradTest, EluGrad) { TensorShape shape({5, 2}); auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index ccb3a231bb..7066f28883 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.python.compat import compat from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops @@ -283,8 +284,9 @@ class LeakyReluTest(test.TestCase): np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), alpha=0.1, use_gpu=True) - # The gradient test for ReLU is a bit tricky as the derivative is not well - # defined at around zero and we want to avoid that in terms of input values. + # The gradient test for Leaky ReLU is a bit tricky as the derivative is not + # well defined at around zero and we want to avoid that in terms of input + # values. def testGradientFloat32(self): with self.test_session(): x = constant_op.constant( @@ -319,39 +321,41 @@ class LeakyReluTest(test.TestCase): self.assertLess(err, 1e-10) def testGradGradFloat32(self): - with self.test_session(): - x = constant_op.constant( - [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], - shape=[2, 5], - name="x") - y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") - z = gradients_impl.gradients(y, x) - x_init = np.asarray( - [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], - dtype=np.float32, - order="F") - err = gradient_checker.compute_gradient_error( - x, [2, 5], z[0], [2, 5], x_init_value=x_init) - print("leaky_relu (float32) gradient of gradient err = ", err) - self.assertLess(err, 1e-4) + with compat.forward_compatibility_horizon(2018, 10, 2): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + name="x") + y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float32, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) + print("leaky_relu (float32) gradient of gradient err = ", err) + self.assertLess(err, 1e-4) def testGradGradFloat64(self): - with self.test_session(): - x = constant_op.constant( - [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], - shape=[2, 5], - dtype=dtypes.float64, - name="x") - y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu") - z = gradients_impl.gradients(y, x) - x_init = np.asarray( - [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], - dtype=np.float64, - order="F") - err = gradient_checker.compute_gradient_error( - x, [2, 5], z[0], [2, 5], x_init_value=x_init) - print("leaky_relu (float64) gradient of gradient err = ", err) - self.assertLess(err, 1e-10) + with compat.forward_compatibility_horizon(2018, 10, 2): + with self.test_session(): + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + dtype=dtypes.float64, + name="x") + y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float64, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) + print("leaky_relu (float64) gradient of gradient err = ", err) + self.assertLess(err, 1e-10) def testGradientScalar(self): with self.test_session() as sess: diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 31b8f3945d..52ea202636 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1601,7 +1601,10 @@ def leaky_relu(features, alpha=0.2, name=None): features = ops.convert_to_tensor(features, name="features") if features.dtype.is_integer: features = math_ops.to_float(features) - return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name) + if compat.forward_compatible(2018, 10, 1): + return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name) + alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha") + return math_ops.maximum(alpha * features, features, name=name) def _flatten_outer_dims(logits): -- GitLab From 2586eb3bfeeef3af357e438ae5aff92d2bac12a5 Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Mon, 3 Sep 2018 11:48:35 +0800 Subject: [PATCH 0030/1085] Code fix against ci_build error results. --- tensorflow/cc/gradients/nn_grad_test.cc | 3 +- tensorflow/core/kernels/relu_op.cc | 8 +-- tensorflow/core/kernels/relu_op.h | 8 +-- tensorflow/core/kernels/relu_op_functor.h | 1 - .../python/kernel_tests/relu_op_test.py | 50 +++++++++---------- .../tools/api/golden/v1/tensorflow.pbtxt | 4 ++ 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index bf0db1f59d..d8c2a1a0fc 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -180,8 +180,7 @@ TEST_F(NNGradTest, LeakyReluGradGrad) { // Avoid input values where Leaky ReLU gradient is not well defined (around // zero). Tensor x_init_value = test::AsTensor( - {2.3f, 1.9f, 1.5f, 1.1f, 0.7f, 0.3f, -0.1f, -0.5f, -0.9f, -1.3f}, - {5, 2}); + {2.3f, 1.9f, 1.5f, 1.1f, 0.7f, 0.3f, -0.1f, -0.5f, -0.9f, -1.3f}, {5, 2}); Tensor features = test::AsTensor( {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, {5, 2}); diff --git a/tensorflow/core/kernels/relu_op.cc b/tensorflow/core/kernels/relu_op.cc index c4f2ef5632..cafa49cbb6 100644 --- a/tensorflow/core/kernels/relu_op.cc +++ b/tensorflow/core/kernels/relu_op.cc @@ -106,15 +106,15 @@ namespace functor { \ template <> \ void LeakyRelu::operator()( \ - const GPUDevice& d, typename TTypes::ConstTensor features, \ - T alpha, typename TTypes::Tensor activations); \ + const GPUDevice& d, typename TTypes::ConstTensor features, T alpha, \ + typename TTypes::Tensor activations); \ extern template struct LeakyRelu; \ \ template <> \ void LeakyReluGrad::operator()( \ const GPUDevice& d, typename TTypes::ConstTensor gradients, \ - typename TTypes::ConstTensor features, \ - T alpha, typename TTypes::Tensor backprops); \ + typename TTypes::ConstTensor features, T alpha, \ + typename TTypes::Tensor backprops); \ extern template struct LeakyReluGrad; \ \ template <> \ diff --git a/tensorflow/core/kernels/relu_op.h b/tensorflow/core/kernels/relu_op.h index c55190065c..fa79ab03ae 100644 --- a/tensorflow/core/kernels/relu_op.h +++ b/tensorflow/core/kernels/relu_op.h @@ -143,8 +143,8 @@ class LeakyReluOp : public UnaryElementWiseOp> { void Operate(OpKernelContext* context, const Tensor& input, Tensor* output) { functor::LeakyRelu functor; - functor(context->eigen_device(), input.flat(), - alpha_, output->flat()); + functor(context->eigen_device(), input.flat(), alpha_, + output->flat()); } private: @@ -183,7 +183,9 @@ class LeakyReluGradOp template void LeakyReluGradOp::OperateNoTemplate(OpKernelContext* context, - const Tensor& g, const Tensor& a, T alpha, Tensor* output) { + const Tensor& g, + const Tensor& a, T alpha, + Tensor* output) { if (!ReluHelpers::ValidateSameSize(context, g, a)) return; functor::LeakyReluGrad functor; functor(context->eigen_device(), g.flat(), a.flat(), alpha, diff --git a/tensorflow/core/kernels/relu_op_functor.h b/tensorflow/core/kernels/relu_op_functor.h index 7f0951451d..548d5a277d 100644 --- a/tensorflow/core/kernels/relu_op_functor.h +++ b/tensorflow/core/kernels/relu_op_functor.h @@ -91,7 +91,6 @@ struct Relu6Grad { } }; - // Functor used by LeakyReluOp to do the computations. template struct LeakyRelu { diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index 7066f28883..3e24b8a2c4 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -323,37 +323,37 @@ class LeakyReluTest(test.TestCase): def testGradGradFloat32(self): with compat.forward_compatibility_horizon(2018, 10, 2): with self.test_session(): - x = constant_op.constant( - [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], - shape=[2, 5], - name="x") - y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") - z = gradients_impl.gradients(y, x) - x_init = np.asarray( - [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], - dtype=np.float32, - order="F") - err = gradient_checker.compute_gradient_error( - x, [2, 5], z[0], [2, 5], x_init_value=x_init) + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + name="x") + y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float32, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4) def testGradGradFloat64(self): with compat.forward_compatibility_horizon(2018, 10, 2): with self.test_session(): - x = constant_op.constant( - [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], - shape=[2, 5], - dtype=dtypes.float64, - name="x") - y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu") - z = gradients_impl.gradients(y, x) - x_init = np.asarray( - [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], - dtype=np.float64, - order="F") - err = gradient_checker.compute_gradient_error( - x, [2, 5], z[0], [2, 5], x_init_value=x_init) + x = constant_op.constant( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + shape=[2, 5], + dtype=dtypes.float64, + name="x") + y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu") + z = gradients_impl.gradients(y, x) + x_init = np.asarray( + [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], + dtype=np.float64, + order="F") + err = gradient_checker.compute_gradient_error( + x, [2, 5], z[0], [2, 5], x_init_value=x_init) print("leaky_relu (float64) gradient of gradient err = ", err) self.assertLess(err, 1e-10) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 4de662fe33..9e8d320f06 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -1324,6 +1324,10 @@ tf_module { name: "lbeta" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "leaky_relu" + argspec: "args=[\'features\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.2\', \'None\'], " + } member_method { name: "less" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " -- GitLab From d2ad105d2dff3c79d8f49f5fb8ce74c38f424e74 Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Mon, 3 Sep 2018 12:10:51 +0800 Subject: [PATCH 0031/1085] Add XLA support for LeakyReluOp. Code contributed by: Meng Chen --- tensorflow/compiler/tests/binary_ops_test.py | 7 ++++ tensorflow/compiler/tests/unary_ops_test.py | 5 +++ tensorflow/compiler/tf2xla/kernels/relu_op.cc | 42 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 0aafda7fb4..8941dd4e27 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -178,6 +178,13 @@ class BinaryOpsTest(xla_test.XLATestCase): [0, 0, 0, 0, 0, 0.1, 0.3, 0.5, 0.7, 0.9, 6.1, 10.0], dtype=dtype), expected=np.array([0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 0, 0], dtype=dtype)) + self._testBinary( + gen_nn_ops._leaky_relu_grad, + np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype), + np.array( + [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], dtype=dtype), + expected=np.array([0.2, 0.4, 0.6, 0.8, 1, 6, 7, 8, 9, 10], dtype=dtype)) + self._testBinary( gen_nn_ops.softmax_cross_entropy_with_logits, np.array([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=dtype), diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index 73adb0d243..91f876fa23 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -361,6 +361,11 @@ class UnaryOpsTest(xla_test.XLATestCase): np.array([[-0.05, 6.05, 5]], dtype=dtype), expected=np.array([[0, 6, 5]], dtype=dtype)) + self._assertOpOutputMatchesExpected( + nn_ops.leaky_relu, + np.array([[-1.0, 1.0]], dtype=dtype), + expected=np.array([[-0.2, 1.0]], dtype=dtype)) + self._assertOpOutputMatchesExpected( nn_ops.softmax, np.array([1, 2, 3, 4], dtype=dtype), diff --git a/tensorflow/compiler/tf2xla/kernels/relu_op.cc b/tensorflow/compiler/tf2xla/kernels/relu_op.cc index d35777ccb1..ec14735884 100644 --- a/tensorflow/compiler/tf2xla/kernels/relu_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/relu_op.cc @@ -50,6 +50,24 @@ class Relu6Op : public XlaOpKernel { } }; + +class LeakyReluOp : public XlaOpKernel { + public: + explicit LeakyReluOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("alpha", &alpha_)); + } + // Compute the max of the input x and alpha*x. + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* builder = ctx->builder(); + auto alpha = XlaHelpers::FloatLiteral(builder, input_type(0), + static_cast(alpha_)); + ctx->SetOutput(0, + xla::Max(xla::Mul(alpha, ctx->Input(0)), ctx->Input(0))); + } + private: + float alpha_; +}; + class ReluGradOp : public XlaOpKernel { public: explicit ReluGradOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} @@ -84,10 +102,34 @@ class Relu6GradOp : public XlaOpKernel { } }; +class LeakyReluGradOp : public XlaOpKernel { + public: + explicit LeakyReluGradOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("alpha", &alpha_)); + } + // Return the lhs (incoming gradient) if the rhs (input feature) > 0, + // otherwise return the alpha * lhs. + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + const TensorShape shape = ctx->InputShape(0); + const auto zero = + xla::Broadcast(XlaHelpers::Zero(b, input_type(0)), shape.dim_sizes()); + const auto pred = xla::Gt(ctx->Input(1), zero); + auto alpha = XlaHelpers::FloatLiteral(b, input_type(0), + static_cast(alpha_)); + ctx->SetOutput(0, + xla::Select(pred, ctx->Input(0), xla::Mul(alpha, ctx->Input(0)))); + } + private: + float alpha_; +}; + REGISTER_XLA_OP(Name("Relu"), ReluOp); REGISTER_XLA_OP(Name("Relu6"), Relu6Op); +REGISTER_XLA_OP(Name("LeakyRelu"), LeakyReluOp); REGISTER_XLA_OP(Name("ReluGrad"), ReluGradOp); REGISTER_XLA_OP(Name("Relu6Grad"), Relu6GradOp); +REGISTER_XLA_OP(Name("LeakyReluGrad"), LeakyReluGradOp); } // namespace } // namespace tensorflow -- GitLab From fa20b59b920233d35bb8da3fbc3c234c369a8291 Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Tue, 4 Sep 2018 14:20:40 -0700 Subject: [PATCH 0032/1085] Move CUDA-specific occupancy calculation into proper file -Maintain functionality, just move CalculateOccupancy() and CompareOccupancy() methods from device_description to cuda_gpu_executor -Remove CUDA requirement in general class device_description --- .../stream_executor/cuda/cuda_gpu_executor.cc | 37 +++++++++++++++++++ .../stream_executor/cuda/cuda_gpu_executor.h | 11 ++++++ .../stream_executor/device_description.cc | 32 ---------------- .../stream_executor/device_description.h | 17 --------- 4 files changed, 48 insertions(+), 49 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index 39b0696c93..458c0e3030 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -490,6 +490,43 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, } } +// Compute and return maximum blocks per core (occupancy) based on the +// device description, some kernel characteristics and the number of threads per +// block. If unable to compute occupancy, zero is returned. +int CalculateOccupancy(const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func) { + int suggested_blocks = 0; + int suggested_threads = 0; + CUresult err = + cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, + func, NULL, shared_memory_per_block, 0); + CHECK_EQ(err, CUDA_SUCCESS); + return suggested_blocks; +} + +// Compute and return the suggested thread count to acheive ideal occupancy. +// If the provided thread dimensions match this number, zero is returned. +int CompareOccupancy(int* initial_blocks, + const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func) { + int suggested_blocks = 0; + int suggested_threads = 0; + CUresult err = + cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, + func, NULL, shared_memory_per_block, 0); + CHECK_EQ(err, CUDA_SUCCESS); + if (suggested_blocks > *initial_blocks) { + *initial_blocks = suggested_blocks; + return suggested_threads; + } else { + return 0; + } +} + void *CUDAExecutor::Allocate(uint64 size) { return CUDADriver::DeviceAllocate(context_, size); } diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h index 8a954d5461..e8ebbc3220 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h @@ -70,6 +70,17 @@ class CUDAExecutor : public internal::StreamExecutorInterface { const BlockDim &block_dims, const KernelBase &k, const KernelArgsArrayBase &args) override; + int CalculateOccupancy(const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func); + + int CompareOccupancy(int* initial_blocks, + const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func); + void *Allocate(uint64 size) override; void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes, diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc index df52ce6cce..726c4adf74 100644 --- a/tensorflow/stream_executor/device_description.cc +++ b/tensorflow/stream_executor/device_description.cc @@ -157,36 +157,4 @@ static uint64 RoundDown(uint64 value, uint64 n) { return port::MathUtil::FloorOfRatio(value, n) * n; } -int CalculateOccupancy(const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func) { - int suggested_blocks = 0; - int suggested_threads = 0; - CUresult err = - cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, - func, NULL, shared_memory_per_block, 0); - CHECK_EQ(err, CUDA_SUCCESS); - return suggested_blocks; -} - -int CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func) { - int suggested_blocks = 0; - int suggested_threads = 0; - CUresult err = - cuOccupancyMaxPotentialBlockSize(&suggested_blocks, &suggested_threads, - func, NULL, shared_memory_per_block, 0); - CHECK_EQ(err, CUDA_SUCCESS); - if (suggested_blocks > *initial_blocks) { - *initial_blocks = suggested_blocks; - return suggested_threads; - } else { - return 0; - } -} - } // namespace stream_executor diff --git a/tensorflow/stream_executor/device_description.h b/tensorflow/stream_executor/device_description.h index d335b9b875..b15ce31216 100644 --- a/tensorflow/stream_executor/device_description.h +++ b/tensorflow/stream_executor/device_description.h @@ -24,7 +24,6 @@ limitations under the License. #include #include "tensorflow/stream_executor/platform/port.h" -#include "tensorflow/stream_executor/cuda/cuda_driver.h" #include "tensorflow/stream_executor/launch_dim.h" #include "tensorflow/stream_executor/platform/port.h" @@ -324,22 +323,6 @@ void CalculateDimensionality(const DeviceDescription &device_description, uint64 element_count, uint64 *threads_per_block, uint64 *block_count); -// Compute and return maximum blocks per core (occupancy) based on the -// device description, some kernel characteristics and the number of threads per -// block. If unable to compute occupancy, zero is returned. -int CalculateOccupancy(const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func); - -// Compute and return the suggested thread count to acheive ideal occupancy. -// If the provided thread dimensions match this number, zero is returned. -int CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func); - } // namespace stream_executor #endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_ -- GitLab From cd6597b8fcd82b51ddb47a297972a1614c2a5d78 Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Tue, 4 Sep 2018 16:17:40 -0700 Subject: [PATCH 0033/1085] Fixed transition typo --- tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index 458c0e3030..a961e9a6c4 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -493,7 +493,7 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, // Compute and return maximum blocks per core (occupancy) based on the // device description, some kernel characteristics and the number of threads per // block. If unable to compute occupancy, zero is returned. -int CalculateOccupancy(const DeviceDescription& device_description, +int CUDAExecutor::CalculateOccupancy(const DeviceDescription& device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, const ThreadDim& thread_dims, CUfunction func) { @@ -508,7 +508,7 @@ int CalculateOccupancy(const DeviceDescription& device_description, // Compute and return the suggested thread count to acheive ideal occupancy. // If the provided thread dimensions match this number, zero is returned. -int CompareOccupancy(int* initial_blocks, +int CUDAExecutor::CompareOccupancy(int* initial_blocks, const DeviceDescription& device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, -- GitLab From 475b7715f16ad0f94fa9986a0eefc1b2cf2044bd Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Tue, 4 Sep 2018 16:31:01 -0700 Subject: [PATCH 0034/1085] Recommended typo fix --- tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index a961e9a6c4..ce2f1ce3ae 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -483,7 +483,7 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, CompareOccupancy(&blocks_per_sm, device_description, regs_per_thread, smem_per_block, thread_dims, cufunc); if (suggested_threads != 0) { - VLOG(2) << "The cuda occupancy calculator reccommends using " + VLOG(2) << "The cuda occupancy calculator recommends using " << suggested_threads << " threads per block to acheive an occupancy of " << blocks_per_sm << " blocks per SM."; -- GitLab From a95281ce1b449d8f92a3799ff9c1dbf661b70bc4 Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Wed, 5 Sep 2018 09:02:40 +0800 Subject: [PATCH 0035/1085] Avoid golden API file changing. --- tensorflow/cc/gradients/nn_grad_test.cc | 3 +-- tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt | 1 + tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index d8c2a1a0fc..f5a09e09dc 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -42,7 +42,6 @@ using ops::MaxPoolV2; using ops::Placeholder; using ops::Relu; using ops::Relu6; -using ops::LeakyRelu; using ops::Selu; using ops::Softmax; using ops::Softplus; @@ -165,7 +164,7 @@ TEST_F(NNGradTest, Relu6Grad) { TEST_F(NNGradTest, LeakyReluGrad) { TensorShape shape({5, 2}); auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); - auto y = LeakyRelu(scope_, x); + auto y = ops::internal::LeakyRelu(scope_, x); // Avoid input values where Leaky ReLU gradient is not well defined (around // zero). Tensor x_init_value = test::AsTensor( diff --git a/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt index 4a61889f54..280148e032 100644 --- a/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_LeakyRelu.pbtxt @@ -1,4 +1,5 @@ op { graph_op_name: "LeakyRelu" + visibility: HIDDEN summary: "Computes rectified linear: `max(features, features * alpha)`." } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 9e8d320f06..4de662fe33 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -1324,10 +1324,6 @@ tf_module { name: "lbeta" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "leaky_relu" - argspec: "args=[\'features\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.2\', \'None\'], " - } member_method { name: "less" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " -- GitLab From d0574f6b25ab01052e093ab92612520a7e4ada8d Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Thu, 6 Sep 2018 08:22:37 -0700 Subject: [PATCH 0036/1085] Fixed clang formatting --- .../stream_executor/cuda/cuda_gpu_executor.cc | 17 +++++++++-------- .../stream_executor/cuda/cuda_gpu_executor.h | 12 ++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index ce2f1ce3ae..ef84d01a94 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -493,10 +493,10 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, // Compute and return maximum blocks per core (occupancy) based on the // device description, some kernel characteristics and the number of threads per // block. If unable to compute occupancy, zero is returned. -int CUDAExecutor::CalculateOccupancy(const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func) { +int CUDAExecutor::CalculateOccupancy( + const DeviceDescription& device_description, uint64 registers_per_thread, + uint64 shared_memory_per_block, const ThreadDim& thread_dims, + CUfunction func) { int suggested_blocks = 0; int suggested_threads = 0; CUresult err = @@ -509,10 +509,11 @@ int CUDAExecutor::CalculateOccupancy(const DeviceDescription& device_description // Compute and return the suggested thread count to acheive ideal occupancy. // If the provided thread dimensions match this number, zero is returned. int CUDAExecutor::CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func) { + const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, + CUfunction func) { int suggested_blocks = 0; int suggested_threads = 0; CUresult err = diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h index e8ebbc3220..1481dcc19a 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h @@ -71,16 +71,16 @@ class CUDAExecutor : public internal::StreamExecutorInterface { const KernelArgsArrayBase &args) override; int CalculateOccupancy(const DeviceDescription& device_description, + uint64 registers_per_thread, + uint64 shared_memory_per_block, + const ThreadDim& thread_dims, CUfunction func); + + int CompareOccupancy(int* initial_blocks, + const DeviceDescription& device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, const ThreadDim& thread_dims, CUfunction func); - int CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, - uint64 registers_per_thread, - uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func); - void *Allocate(uint64 size) override; void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes, -- GitLab From e3654a3cb4e26c26409aeeb9e127e3addcb14cee Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 6 Sep 2018 19:20:11 +0000 Subject: [PATCH 0037/1085] Add float16 support on GPU for tf.contrib.image.transform This fix tries to address the issue raised in 22115 where there were no float16 support on GPU for tf.contrib.image.transform. This fix fixes 22115. Signed-off-by: Yong Tang --- tensorflow/contrib/image/kernels/image_ops.cc | 2 ++ tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/tensorflow/contrib/image/kernels/image_ops.cc b/tensorflow/contrib/image/kernels/image_ops.cc index 370a8caf6a..788bf04b28 100644 --- a/tensorflow/contrib/image/kernels/image_ops.cc +++ b/tensorflow/contrib/image/kernels/image_ops.cc @@ -156,6 +156,7 @@ namespace functor { TF_CALL_uint8(DECLARE_FUNCTOR); TF_CALL_int32(DECLARE_FUNCTOR); TF_CALL_int64(DECLARE_FUNCTOR); +TF_CALL_half(DECLARE_FUNCTOR); TF_CALL_float(DECLARE_FUNCTOR); TF_CALL_double(DECLARE_FUNCTOR); @@ -175,6 +176,7 @@ TF_CALL_double(DECLARE_FUNCTOR); TF_CALL_uint8(REGISTER); TF_CALL_int32(REGISTER); TF_CALL_int64(REGISTER); +TF_CALL_half(REGISTER); TF_CALL_float(REGISTER); TF_CALL_double(REGISTER); diff --git a/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc b/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc index 8743a5ff72..36b9a236a6 100644 --- a/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc +++ b/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc @@ -32,6 +32,7 @@ typedef Eigen::GpuDevice GPUDevice; template class FillProjectiveTransform; template class FillProjectiveTransform; template class FillProjectiveTransform; +template class FillProjectiveTransform; template class FillProjectiveTransform; template class FillProjectiveTransform; -- GitLab From 7d7e8a725aeede4b724f7376d22df2c7f2ebdcf9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 6 Sep 2018 19:22:39 +0000 Subject: [PATCH 0038/1085] Add test case for float16 support on GPU for tf.contrib.image.transform Signed-off-by: Yong Tang --- .../contrib/image/python/kernel_tests/image_ops_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py index 376c0751ee..ef1f79bb94 100644 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py @@ -272,6 +272,13 @@ class ImageOpsTest(test_util.TensorFlowTestCase): with self.cached_session(): self.assertAllEqual([[[[1], [0]], [[0], [1]]]], result.eval()) + def test_transform_data_types(self): + for dtype in _DTYPES: + image = constant_op.constant([[1, 2], [3, 4]], dtype=dtype) + value = image_ops.transform(image, [1] * 8) + with self.test_session(use_gpu=True): + self.assertAllEqual(value.eval(), np.array([[4, 4], [4, 4]]).astype(dtype.as_numpy_dtype())) + class BipartiteMatchTest(test_util.TensorFlowTestCase): -- GitLab From 04e20965487c36f43ba5c773b547b23e39478a5c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 6 Sep 2018 19:25:22 +0000 Subject: [PATCH 0039/1085] Pylint fix Signed-off-by: Yong Tang --- .../contrib/image/python/kernel_tests/image_ops_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py index ef1f79bb94..4997c31a7f 100644 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py @@ -277,7 +277,9 @@ class ImageOpsTest(test_util.TensorFlowTestCase): image = constant_op.constant([[1, 2], [3, 4]], dtype=dtype) value = image_ops.transform(image, [1] * 8) with self.test_session(use_gpu=True): - self.assertAllEqual(value.eval(), np.array([[4, 4], [4, 4]]).astype(dtype.as_numpy_dtype())) + self.assertAllEqual( + value.eval(), + np.array([[4, 4], [4, 4]]).astype(dtype.as_numpy_dtype())) class BipartiteMatchTest(test_util.TensorFlowTestCase): -- GitLab From 6a5090b086bc9d665eb9e65f05eb94cdb58baaa2 Mon Sep 17 00:00:00 2001 From: Matt Conley Date: Thu, 6 Sep 2018 13:09:12 -0700 Subject: [PATCH 0040/1085] Fully fixed clang errors --- tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 12 ++++++------ tensorflow/stream_executor/cuda/cuda_gpu_executor.h | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index ef84d01a94..9d5bcc7f77 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -472,7 +472,7 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, const DeviceDescription &device_description = kernel.parent()->GetDeviceDescription(); - const CUDAKernel* cuda_kernel = AsCUDAKernel(&kernel); + const CUDAKernel *cuda_kernel = AsCUDAKernel(&kernel); CUfunction cufunc = cuda_kernel->AsCUDAFunctionValue(); int blocks_per_sm = CalculateOccupancy(device_description, regs_per_thread, @@ -494,8 +494,8 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel, // device description, some kernel characteristics and the number of threads per // block. If unable to compute occupancy, zero is returned. int CUDAExecutor::CalculateOccupancy( - const DeviceDescription& device_description, uint64 registers_per_thread, - uint64 shared_memory_per_block, const ThreadDim& thread_dims, + const DeviceDescription &device_description, uint64 registers_per_thread, + uint64 shared_memory_per_block, const ThreadDim &thread_dims, CUfunction func) { int suggested_blocks = 0; int suggested_threads = 0; @@ -508,11 +508,11 @@ int CUDAExecutor::CalculateOccupancy( // Compute and return the suggested thread count to acheive ideal occupancy. // If the provided thread dimensions match this number, zero is returned. -int CUDAExecutor::CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, +int CUDAExecutor::CompareOccupancy(int *initial_blocks, + const DeviceDescription &device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, - const ThreadDim& thread_dims, + const ThreadDim &thread_dims, CUfunction func) { int suggested_blocks = 0; int suggested_threads = 0; diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h index 1481dcc19a..53b2a29ae7 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h @@ -70,16 +70,16 @@ class CUDAExecutor : public internal::StreamExecutorInterface { const BlockDim &block_dims, const KernelBase &k, const KernelArgsArrayBase &args) override; - int CalculateOccupancy(const DeviceDescription& device_description, + int CalculateOccupancy(const DeviceDescription &device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func); + const ThreadDim &thread_dims, CUfunction func); - int CompareOccupancy(int* initial_blocks, - const DeviceDescription& device_description, + int CompareOccupancy(int *initial_blocks, + const DeviceDescription &device_description, uint64 registers_per_thread, uint64 shared_memory_per_block, - const ThreadDim& thread_dims, CUfunction func); + const ThreadDim &thread_dims, CUfunction func); void *Allocate(uint64 size) override; -- GitLab From e25cf78285fef5234380ee26fef9090a939e91f5 Mon Sep 17 00:00:00 2001 From: Richard Yu Date: Thu, 6 Sep 2018 17:05:08 -0700 Subject: [PATCH 0041/1085] Ensure all ValueErrors are raised --- tensorflow/contrib/quantize/python/fold_batch_norms.py | 2 +- tensorflow/python/keras/layers/embeddings.py | 8 ++++---- tensorflow/python/ops/nn_ops.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index d9f179bee4..d882b79892 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -628,7 +628,7 @@ def _GetBatchNormParams(graph, context, has_scaling): bn_decay_var_tensor = _FindMatchingTensor(graph, op_suffix_bn_decay_var, context) if batch_mean_tensor is None and moving_mean_tensor is None: - ValueError('Error folding unfused batch norms') + raise ValueError('Error folding unfused batch norms') if has_scaling: gamma_tensor = _FindMatchingTensor(graph, op_suffix_gamma, context) diff --git a/tensorflow/python/keras/layers/embeddings.py b/tensorflow/python/keras/layers/embeddings.py index 629a9ec9a1..a0b9393812 100644 --- a/tensorflow/python/keras/layers/embeddings.py +++ b/tensorflow/python/keras/layers/embeddings.py @@ -142,13 +142,13 @@ class Embedding(Layer): else: in_lens = [self.input_length] if len(in_lens) != len(input_shape) - 1: - ValueError('"input_length" is %s, but received input has shape %s' % - (str(self.input_length), str(input_shape))) + raise ValueError('"input_length" is %s, but received input has shape %s' % + (str(self.input_length), str(input_shape))) else: for i, (s1, s2) in enumerate(zip(in_lens, input_shape[1:])): if s1 is not None and s2 is not None and s1 != s2: - ValueError('"input_length" is %s, but received input has shape %s' % - (str(self.input_length), str(input_shape))) + raise ValueError('"input_length" is %s, but received input has shape %s' % + (str(self.input_length), str(input_shape))) elif s1 is None: in_lens[i] = s2 return (input_shape[0],) + tuple(in_lens) + (self.output_dim,) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index ef9afd9e8e..17e10995f2 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -427,8 +427,8 @@ class _WithSpaceToBatch(object): try: input_shape.with_rank_at_least(expected_input_rank) except ValueError: - ValueError("input tensor must have rank %d at least" % - (expected_input_rank)) + raise ValueError("input tensor must have rank %d at least" % + (expected_input_rank)) const_rate = tensor_util.constant_value(dilation_rate) rate_or_const_rate = dilation_rate @@ -818,12 +818,12 @@ class Convolution(object): try: input_shape.with_rank(num_spatial_dims + 2) except ValueError: - ValueError("input tensor must have rank %d" % (num_spatial_dims + 2)) + raise ValueError("input tensor must have rank %d" % (num_spatial_dims + 2)) try: filter_shape.with_rank(num_spatial_dims + 2) except ValueError: - ValueError("filter tensor must have rank %d" % (num_spatial_dims + 2)) + raise ValueError("filter tensor must have rank %d" % (num_spatial_dims + 2)) if data_format is None or not data_format.startswith("NC"): input_channels_dim = input_shape[num_spatial_dims + 1] -- GitLab From 864e290d1776895d7877777b8368ca8bc6fc22a3 Mon Sep 17 00:00:00 2001 From: Edvard Fagerholm Date: Wed, 29 Aug 2018 11:56:35 +0300 Subject: [PATCH 0042/1085] Make tf.transpose emit simpler graph when possible If not given an explicit 'perm' parameter, tf.transpose currently emits a graph that dynamically calculates it from the rank of the input tensor. This is completely unnecessary when the rank of the input can be statically determined at graph construction time. Modify tf.transpose to emit 'perm' as a single Const node whenever possible. --- tensorflow/python/ops/array_ops.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 7bf3869ddf..9597839301 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1409,8 +1409,13 @@ def transpose(a, perm=None, name="transpose", conjugate=False): gen_array_ops.conjugate_transpose if (conjugate and a.dtype.is_complex) else gen_array_ops.transpose) if perm is None: - rank = gen_array_ops.rank(a) - perm = (rank - 1) - gen_math_ops._range(0, rank, 1) + a = ops.convert_to_tensor(a, name="a") + if not a.get_shape().ndims: + rank = gen_array_ops.rank(a) + perm = (rank - 1) - gen_math_ops._range(0, rank, 1) + else: + rank = a.get_shape().ndims + perm = (rank - 1) - np.arange(rank) ret = transpose_fn(a, perm, name=name) # NOTE(mrry): Setting the shape explicitly because # reverse is not handled by the shape function. -- GitLab From 90cf7fb7786c8a9c135ef73482856b082e80f61a Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Tue, 11 Sep 2018 12:48:30 +0800 Subject: [PATCH 0043/1085] Fix lint errors and typos. --- tensorflow/compiler/tests/binary_ops_test.py | 9 +++++---- tensorflow/compiler/tf2xla/kernels/relu_op.cc | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 8941dd4e27..069e83d083 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -179,11 +179,12 @@ class BinaryOpsTest(xla_test.XLATestCase): expected=np.array([0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 0, 0], dtype=dtype)) self._testBinary( - gen_nn_ops._leaky_relu_grad, + gen_nn_ops.leaky_relu_grad, np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype), - np.array( - [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], dtype=dtype), - expected=np.array([0.2, 0.4, 0.6, 0.8, 1, 6, 7, 8, 9, 10], dtype=dtype)) + np.array([-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], + dtype=dtype), + expected=np.array([0.2, 0.4, 0.6, 0.8, 1, 6, 7, 8, 9, 10], + dtype=dtype)) self._testBinary( gen_nn_ops.softmax_cross_entropy_with_logits, diff --git a/tensorflow/compiler/tf2xla/kernels/relu_op.cc b/tensorflow/compiler/tf2xla/kernels/relu_op.cc index ec14735884..8d65e0339c 100644 --- a/tensorflow/compiler/tf2xla/kernels/relu_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/relu_op.cc @@ -50,7 +50,6 @@ class Relu6Op : public XlaOpKernel { } }; - class LeakyReluOp : public XlaOpKernel { public: explicit LeakyReluOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { @@ -61,9 +60,9 @@ class LeakyReluOp : public XlaOpKernel { xla::XlaBuilder* builder = ctx->builder(); auto alpha = XlaHelpers::FloatLiteral(builder, input_type(0), static_cast(alpha_)); - ctx->SetOutput(0, - xla::Max(xla::Mul(alpha, ctx->Input(0)), ctx->Input(0))); + ctx->SetOutput(0, xla::Max(xla::Mul(alpha, ctx->Input(0)), ctx->Input(0))); } + private: float alpha_; }; @@ -115,11 +114,12 @@ class LeakyReluGradOp : public XlaOpKernel { const auto zero = xla::Broadcast(XlaHelpers::Zero(b, input_type(0)), shape.dim_sizes()); const auto pred = xla::Gt(ctx->Input(1), zero); - auto alpha = XlaHelpers::FloatLiteral(b, input_type(0), - static_cast(alpha_)); - ctx->SetOutput(0, - xla::Select(pred, ctx->Input(0), xla::Mul(alpha, ctx->Input(0)))); + auto alpha = + XlaHelpers::FloatLiteral(b, input_type(0), static_cast(alpha_)); + ctx->SetOutput( + 0, xla::Select(pred, ctx->Input(0), xla::Mul(alpha, ctx->Input(0)))); } + private: float alpha_; }; -- GitLab From 8530167f68673fa756565c0394bbe2dcdc39db05 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Fri, 24 Aug 2018 16:52:07 +0300 Subject: [PATCH 0044/1085] Add IgniteDataset that allows to work with Apache Ignite. --- configure.py | 2 + tensorflow/BUILD | 6 + tensorflow/contrib/BUILD | 15 + tensorflow/contrib/cmake/python_modules.txt | 2 + tensorflow/contrib/ignite/BUILD | 136 ++++ tensorflow/contrib/ignite/README.md | 167 ++++ tensorflow/contrib/ignite/__init__.py | 42 + .../kernels/ignite_binary_object_parser.cc | 304 +++++++ .../kernels/ignite_binary_object_parser.h | 54 ++ .../contrib/ignite/kernels/ignite_client.cc | 55 ++ .../contrib/ignite/kernels/ignite_client.h | 40 + .../contrib/ignite/kernels/ignite_dataset.cc | 123 +++ .../contrib/ignite/kernels/ignite_dataset.h | 65 ++ .../ignite/kernels/ignite_dataset_iterator.cc | 447 ++++++++++ .../ignite/kernels/ignite_dataset_iterator.h | 87 ++ .../ignite/kernels/ignite_dataset_ops.cc | 145 ++++ .../ignite/kernels/ignite_plain_client.h | 43 + .../kernels/ignite_plain_client_unix.cc | 132 +++ .../kernels/ignite_plain_client_windows.cc | 143 ++++ .../ignite/kernels/ignite_ssl_wrapper.cc | 149 ++++ .../ignite/kernels/ignite_ssl_wrapper.h | 49 ++ tensorflow/contrib/ignite/ops/dataset_ops.cc | 64 ++ .../ignite/python/ops/ignite_dataset_ops.py | 763 ++++++++++++++++++ .../ignite/python/ops/ignite_op_loader.py | 25 + .../ignite/python/tests/bin/start-plain.sh | 24 + .../ignite/python/tests/bin/start-ssl-auth.sh | 28 + .../ignite/python/tests/bin/start-ssl.sh | 26 + .../tests/config/ignite-config-plain.xml | 39 + .../tests/config/ignite-config-ssl-auth.xml | 59 ++ .../python/tests/config/ignite-config-ssl.xml | 59 ++ .../python/tests/ignite_dataset_test.py | 77 ++ .../ignite/python/tests/keystore/client.jks | Bin 0 -> 3232 bytes .../ignite/python/tests/keystore/client.pem | 69 ++ .../ignite/python/tests/keystore/server.jks | Bin 0 -> 3230 bytes .../ignite/python/tests/keystore/trust.jks | Bin 0 -> 2432 bytes .../contrib/ignite/python/tests/sql/init.sql | 20 + .../ignite/python/tests/start_ignite.sh | 30 + .../ignite/python/tests/stop_ignite.sh | 19 + 38 files changed, 3508 insertions(+) create mode 100644 tensorflow/contrib/ignite/BUILD create mode 100644 tensorflow/contrib/ignite/README.md create mode 100644 tensorflow/contrib/ignite/__init__.py create mode 100644 tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h create mode 100644 tensorflow/contrib/ignite/kernels/ignite_client.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_client.h create mode 100644 tensorflow/contrib/ignite/kernels/ignite_dataset.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_dataset.h create mode 100644 tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h create mode 100644 tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_plain_client.h create mode 100644 tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc create mode 100644 tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h create mode 100644 tensorflow/contrib/ignite/ops/dataset_ops.cc create mode 100644 tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py create mode 100644 tensorflow/contrib/ignite/python/ops/ignite_op_loader.py create mode 100755 tensorflow/contrib/ignite/python/tests/bin/start-plain.sh create mode 100755 tensorflow/contrib/ignite/python/tests/bin/start-ssl-auth.sh create mode 100755 tensorflow/contrib/ignite/python/tests/bin/start-ssl.sh create mode 100644 tensorflow/contrib/ignite/python/tests/config/ignite-config-plain.xml create mode 100644 tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl-auth.xml create mode 100644 tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl.xml create mode 100644 tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py create mode 100644 tensorflow/contrib/ignite/python/tests/keystore/client.jks create mode 100644 tensorflow/contrib/ignite/python/tests/keystore/client.pem create mode 100644 tensorflow/contrib/ignite/python/tests/keystore/server.jks create mode 100644 tensorflow/contrib/ignite/python/tests/keystore/trust.jks create mode 100644 tensorflow/contrib/ignite/python/tests/sql/init.sql create mode 100755 tensorflow/contrib/ignite/python/tests/start_ignite.sh create mode 100755 tensorflow/contrib/ignite/python/tests/stop_ignite.sh diff --git a/configure.py b/configure.py index 361bd4764d..8f1957e870 100644 --- a/configure.py +++ b/configure.py @@ -1502,6 +1502,8 @@ def main(): 'with_aws_support', True, 'aws') set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform', 'with_kafka_support', True, 'kafka') + set_build_var(environ_cp, 'TF_NEED_IGNITE', 'Apache Ignite', + 'with_ignite_support', True, 'ignite') set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', False, 'xla') set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support', diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 386e0096ff..6c29c78793 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -248,6 +248,12 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "with_ignite_support", + define_values = {"with_ignite_support": "true"}, + visibility = ["//visibility:public"], +) + # Crosses between platforms and file system libraries not supported on those # platforms due to limitations in nested select() statements. config_setting( diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 798f499870..f055e643d0 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -118,6 +118,11 @@ py_library( "//tensorflow/contrib/kafka", ], "//conditions:default": [], + }) + select({ + "//tensorflow:with_ignite_support": [ + "//tensorflow/contrib/ignite", + ], + "//conditions:default": [], }) + select({ "//tensorflow:with_aws_support_windows_override": [], "//tensorflow:with_aws_support": [ @@ -160,6 +165,11 @@ cc_library( "//tensorflow/contrib/kafka:dataset_kernels", ], "//conditions:default": [], + }) + select({ + "//tensorflow:with_ignite_support": [ + "//tensorflow/contrib/ignite:dataset_kernels", + ], + "//conditions:default": [], }) + select({ "//tensorflow:with_aws_support_windows_override": [], "//tensorflow:with_aws_support": [ @@ -197,6 +207,11 @@ cc_library( "//tensorflow/contrib/kafka:dataset_ops_op_lib", ], "//conditions:default": [], + }) + select({ + "//tensorflow:with_ignite_support": [ + "//tensorflow/contrib/ignite:dataset_ops_op_lib", + ], + "//conditions:default": [], }) + select({ "//tensorflow:with_aws_support_windows_override": [], "//tensorflow:with_aws_support": [ diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt index fb871acae9..56755e817a 100644 --- a/tensorflow/contrib/cmake/python_modules.txt +++ b/tensorflow/contrib/cmake/python_modules.txt @@ -207,6 +207,8 @@ tensorflow/contrib/integrate/python tensorflow/contrib/integrate/python/ops tensorflow/contrib/kafka/python tensorflow/contrib/kafka/python/ops +tensorflow/contrib/ignite/python +tensorflow/contrib/ignite/python/ops tensorflow/contrib/keras tensorflow/contrib/keras/api tensorflow/contrib/keras/api/keras diff --git a/tensorflow/contrib/ignite/BUILD b/tensorflow/contrib/ignite/BUILD new file mode 100644 index 0000000000..9f6c666893 --- /dev/null +++ b/tensorflow/contrib/ignite/BUILD @@ -0,0 +1,136 @@ +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +load( + "//tensorflow:tensorflow.bzl", + "tf_gen_op_wrapper_py", + "tf_kernel_library", + "tf_custom_op_library", + "tf_custom_op_py_library", + "tf_gen_op_libs", + "tf_py_test", + "if_not_windows", + "if_windows", +) + +py_library( + name = "ignite", + srcs = ["__init__.py"], + srcs_version = "PY2AND3", + deps = [ + ":dataset_ops", + ], +) + +tf_custom_op_library( + name = "_dataset_ops.so", + srcs = ["ops/dataset_ops.cc"], + deps = [":dataset_kernels"], +) + +tf_gen_op_libs( + op_lib_names = ["dataset_ops"], +) + +cc_library( + name = "dataset_kernels", + srcs = [ + "kernels/ignite_dataset_ops.cc", + "kernels/ignite_client.h", + "kernels/ignite_client.cc", + "kernels/ignite_plain_client.h", + "kernels/ignite_ssl_wrapper.h", + "kernels/ignite_ssl_wrapper.cc", + "kernels/ignite_binary_object_parser.h", + "kernels/ignite_binary_object_parser.cc", + "kernels/ignite_dataset.h", + "kernels/ignite_dataset.cc", + "kernels/ignite_dataset_iterator.h", + "kernels/ignite_dataset_iterator.cc", + ] + if_not_windows([ + "kernels/ignite_plain_client_unix.cc", + ]) + if_windows([ + "kernels/ignite_plain_client_windows.cc", + ]), + deps = [ + "//tensorflow/core:framework_headers_lib", + "//third_party/eigen3", + "@boringssl//:ssl", + "@protobuf_archive//:protobuf_headers", + ], + alwayslink = 1, +) + +py_library( + name = "dataset_ops", + srcs = [ + "python/ops/ignite_dataset_ops.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":ignite_op_loader", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:util", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], +) + +tf_gen_op_wrapper_py( + name = "gen_dataset_ops", + out = "python/ops/gen_dataset_ops.py", + deps = ["//tensorflow/contrib/ignite:dataset_ops_op_lib"], +) + +tf_kernel_library( + name = "dataset_ops_kernels", + deps = [ + ":dataset_kernels", + "//tensorflow/core:framework", + ], + alwayslink = 1, +) + +tf_custom_op_py_library( + name = "ignite_op_loader", + srcs = ["python/ops/ignite_op_loader.py"], + dso = ["//tensorflow/contrib/ignite:_dataset_ops.so"], + kernels = [ + ":dataset_ops_kernels", + "//tensorflow/contrib/ignite:dataset_ops_op_lib", + ], + srcs_version = "PY2AND3", + deps = [ + ":gen_dataset_ops", + "//tensorflow/contrib/util:util_py", + "//tensorflow/python:platform", + ], +) + +# The Apache Ignite servers have to setup before the test and tear down +# after the test manually. The docker engine has to be installed. +# +# To setup Apache Ignite servers: +# $ bash ./python/tests/start_ignite.sh +# +# To tear down Apache Ignite servers: +# $ bash ./python/tests/stop_ignite.sh +tf_py_test( + name = "ignite_dataset_test", + srcs = ["python/tests/ignite_dataset_test.py"], + additional_deps = [ + ":ignite", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + ], + tags = [ + "manual", + "no_windows", + "notap", + ], +) diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md new file mode 100644 index 0000000000..9054344e94 --- /dev/null +++ b/tensorflow/contrib/ignite/README.md @@ -0,0 +1,167 @@ +### Ignite Dataset +# Ignite Dataset + +- [Overview](#overview) +- [Features](#features) + * [Distributed In-Memory Datasource](#distributed-in-memory-datasource) + * [Structured Objects](#structured-objects) + * [Distributed Training](#distributed-training) + * [SSL Connection](#ssl-connection) + * [Windows Support](#windows-support) +- [Try it out](#try-it-out) +- [Limitations](#limitations) + +## Overview + +[Apache Ignite](https://ignite.apache.org/) is a memory-centric distributed database, caching, and processing platform for +transactional, analytical, and streaming workloads, delivering in-memory speeds at petabyte scale. This contrib package contains an integration between Apache Ignite and TensorFlow. The integration is based on [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) from TensorFlow side and [Binary Client Protocol](https://apacheignite.readme.io/v2.6/docs/binary-client-protocol) from Apache Ignite side. It allows to use Apache Ignite as a datasource for neural network training, inference and all other computations supported by TensorFlow. + +## Features + +Ignite Dataset provides a set of features that makes it possible to use it in a wide range of cases. The most important and interesting features are described below. + +### Distributed In-Memory Datasource +[Apache Ignite](https://ignite.apache.org/) is a distributed in-memory database, caching, and processing platform that allows to avoid limitations of hard drive and provide high reading speed and ability to store and operate with as much data as you need in distributed cluster. Using of Ignite Dataset makes it possible to utilize all these advantages. +- If you have a **gigabyte** of data you can keep it on a single machine on a hard drive, but you will face with hard drive speed limitations. At the same time, you can store your data in Apache Ignite on the same machine and use it as a datasource for TensorFlow and thus avoid these limitations. +- If you have a **terabyte** of data you probably still can keep it on a single machine on a hard drive, but you will face with hard drive speed limitations again. At the same time, you can store your data in Apache Ignite distributed in-memory cluster and use it as a datasource for TensorFlow and thus avoid these limitations. +- If you have a **petabyte** of data you can't keep it on a single machine. At the same time, you can store your data in Apache Ignite distributed in-memory cluster and use it as a datasource for TensorFlow. + +It's important that Apache Ignite is not just a step of ETL pipeline between database or data warehouse and TensorFlow. Apache Ignite is a high-grade database itself. Choosing Apache Ignite and TensorFlow you are getting everything you need to work with operational or historical data and, in the same time, an ability to use this data for neural network training and inference. + +```bash +$ apache-ignite-fabric/bin/ignite.sh +$ apache-ignite-fabric/bin/sqlline.sh -u "jdbc:ignite:thin://localhost:10800/" + +jdbc:ignite:thin://localhost/> CREATE TABLE KITTEN_CACHE (ID LONG PRIMARY KEY, NAME VARCHAR); +jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (1, 'WARM KITTY'); +jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (2, 'SOFT KITTY'); +jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL OF FUR'); +``` + +```python +>>> import tensorflow as tf +>>> from tensorflow.contrib.ignite import IgniteDataset +>>> +>>> dataset = IgniteDataset(cache_name="SQL_PUBLIC_KITTEN_CACHE") +>>> iterator = dataset.make_one_shot_iterator() +>>> next_obj = iterator.get_next() +>>> +>>> with tf.Session() as sess: +>>> for _ in range(3): +>>> print(sess.run(next_obj)) + +{'key': 1, 'val': {'NAME': b'WARM KITTY'}} +{'key': 2, 'val': {'NAME': b'SOFT KITTY'}} +{'key': 3, 'val': {'NAME': b'LITTLE BALL OF FUR'}} +``` + +### Structured Objects +[Apache Ignite](https://ignite.apache.org/) allows to store any objects you would like to store. These objects can have any hierarchy. Ignite Dataset provides an ability to work with such objects. + +```python +>>> import tensorflow as tf +>>> from tensorflow.contrib.ignite import IgniteDataset +>>> +>>> dataset = IgniteDataset(cache_name="IMAGES") +>>> iterator = dataset.make_one_shot_iterator() +>>> next_obj = iterator.get_next() +>>> +>>> with tf.Session() as sess: +>>> print(sess.run(next_obj)) + +{ + 'key': 'kitten.png', + 'val': { + 'metadata': { + 'file_name': b'kitten.png', + 'label': b'little ball of fur', + width: 800, + height: 600 + }, + 'pixels': [0, 0, 0, 0, ..., 0] + } +} +``` + Neural network training and other computations require transformations that can be done as part of [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) pipeline if you use Ignite Dataset. + +```python +>>> import tensorflow as tf +>>> from tensorflow.contrib.ignite import IgniteDataset +>>> +>>> dataset = IgniteDataset(cache_name="IMAGES").map(lambda obj: obj['val']['pixels']) +>>> iterator = dataset.make_one_shot_iterator() +>>> next_obj = iterator.get_next() +>>> +>>> with tf.Session() as sess: +>>> print(sess.run(next_obj)) + +[0, 0, 0, 0, ..., 0] +``` + +### Distributed Training + +TensorFlow is a machine learning framework that [natively supports](https://www.tensorflow.org/deploy/distributed) distributed neural network training, inference and other computations. The main idea behind the distributed neural network training is an ability to calculate gradients of loss functions (squares of the errors) on every partition of data (in terms of horizontal partitioning) and then sum them to get loss function gradient of the whole dataset. + + + +Utilizing this ability we can calculate gradients on the nodes the data is stored on, reduce them and then finally update model parameters. It allows to avoid data transfers between nodes and thus to avoid network bottleneck. + +Apache Ignite uses horizontal partitioning to store data in distributed cluster. When we create Apache Ignite cache (or table in terms of SQL) we can specify the number of partitions the data will be partitioned on. If, for example, Apache Ignite cluster consists of 10 machines and we creates cache with 10 partitions then every machine will maintain approximately one data partition. + +Ignite Dataset allows to utilize these two aspects of distributed neural network training (using TensorFlow) and Apache Ignite partitioning. Ignite Dataset is a computation graph operation that might be performed on a remote worker. The remote worker can override Ignite Dataset parameters (such as `host`, `port` or `part`) by setting correstondent environment variables for worker process (such as `IGNITE_DATASET_HOST`, `IGNITE_DATASET_PORT` or `IGNITE_DATASET_PART`). Using this overriding approach we are able to assign specific partition to every worker so that one worker handles one partition and, at the same time, transparently work with single dataset. + +```python +>>> import tensorflow as tf +>>> from tensorflow.contrib.ignite import IgniteDataset +>>> +>>> dataset = IgniteDataset("IMAGES") +>>> +>>> # Compute gradients locally on every worker node. +>>> gradients = [] +>>> for i in range(5): +>>> with tf.device("/job:WORKER/task:%d" % i): +>>> device_iterator = dataset.make_one_shot_iterator() +>>> device_next_obj = device_iterator.get_next() +>>> gradient = compute_gradient(device_next_obj) +>>> gradients.append(gradient) +>>> +>>> # Aggregate them on master node. +>>> result_gradient = tf.reduce_sum(gradients) +>>> +>>> with tf.Session("grpc://localhost:10000") as sess: +>>> print(sess.run(result_gradient)) +``` + +High-level TensorFlow API for [distributed training](https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy) is supported as well. + +### SSL Connection + +Your data should not be accessible without any control. Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security) and authentification. Ignite Dataset supports both SSL connection with and without authntication. For more information please see [Apache Ignite SSL/TLS](https://apacheignite.readme.io/docs/ssltls) documentation. + +```python +>>> import tensorflow as tf +>>> from tensorflow.contrib.ignite import IgniteDataset +>>> +>>> dataset = IgniteDataset(cache_name="IMAGES", certfile="client.pem", cert_password="password", username="ignite", password="ignite") +>>> ... +``` + +### Windows Support + +Ignite Dataset is fully compatible with Windows, so you can use it as part of TensorFlow on your Windows workstation as well as on Linux/MacOS systems. + +## Try it out + +The simplest way to try Ignite Dataset out is to run [Docker](https://www.docker.com/) container with Apache Ignite and loaded [MNIST](http://yann.lecun.com/exdb/mnist/) data and then interruct with it using Ignite Dataset. Such container is available on Docker Hub: [dmitrievanthony/ignite-with-mnist](https://hub.docker.com/r/dmitrievanthony/ignite-with-mnist/). You need to start this container on your machine: + +``` +docker run -it -p 10800:10800 dmitrievanthony/ignite-with-mnist +``` + +After that you will be able to work with it following way: + +![ignite-dataset-mnist](https://s3.amazonaws.com/helloworld23423423ew23/ignite-dataset-mnist.png "Ignite Dataset Mnist") + +## Limitations + +Presently Ignite Dataset works with assumption that all objects in the cache have the same structure (homogeneous objects) and the cache contains at least one object. Another limitation concerns structured objects, Ignite Dataset does not support UUID, Maps and Object arrays that might be parts of object structures. \ No newline at end of file diff --git a/tensorflow/contrib/ignite/__init__.py b/tensorflow/contrib/ignite/__init__.py new file mode 100644 index 0000000000..468920a557 --- /dev/null +++ b/tensorflow/contrib/ignite/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Apache Ignite is a memory-centric distributed database, caching, and + processing platform for transactional, analytical, and streaming workloads, + delivering in-memory speeds at petabyte scale. This contrib package + contains an integration between Apache Ignite and TensorFlow. The + integration is based on tf.data from TensorFlow side and Binary Client + Protocol from Apache Ignite side. It allows to use Apache Ignite as a + datasource for neural network training, inference and all other + computations supported by TensorFlow. Ignite Dataset is based on Apache + Ignite Binary Client Protocol: + https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. + +@@IgniteDataset +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.ignite.python.ops.ignite_dataset_ops \ +import IgniteDataset + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + "IgniteDataset", +] + +remove_undocumented(__name__) diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc new file mode 100644 index 0000000000..bf0ef8766e --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc @@ -0,0 +1,304 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_binary_object_parser.h" + +namespace ignite { + +tensorflow::Status BinaryObjectParser::Parse( + uint8_t*& ptr, std::vector& out_tensors, + std::vector& types) { + uint8_t object_type_id = *ptr; + ptr += 1; + + switch (object_type_id) { + case BYTE: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_UINT8, {}); + tensor.scalar()() = *((uint8_t*)ptr); + ptr += 1; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case SHORT: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT16, {}); + tensor.scalar()() = *((int16_t*)ptr); + ptr += 2; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case INT: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT32, {}); + tensor.scalar()() = *((int32_t*)ptr); + ptr += 4; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case LONG: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT64, {}); + tensor.scalar()() = *((int64_t*)ptr); + ptr += 8; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case FLOAT: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_FLOAT, {}); + tensor.scalar()() = *((float*)ptr); + ptr += 4; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case DOUBLE: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_DOUBLE, {}); + tensor.scalar()() = *((double*)ptr); + ptr += 8; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case UCHAR: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_UINT16, {}); + tensor.scalar()() = *((uint16_t*)ptr); + ptr += 2; + out_tensors.emplace_back(std::move(tensor)); + break; + } + case BOOL: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_BOOL, {}); + tensor.scalar()() = *((bool*)ptr); + ptr += 1; + out_tensors.emplace_back(std::move(tensor)); + + break; + } + case STRING: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_STRING, {}); + tensor.scalar()() = std::string((char*)ptr, length); + ptr += length; + out_tensors.emplace_back(std::move(tensor)); + + break; + } + case DATE: { + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT64, {}); + tensor.scalar()() = *((int64_t*)ptr); + ptr += 8; + out_tensors.emplace_back(std::move(tensor)); + + break; + } + case BYTE_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_UINT8, + tensorflow::TensorShape({length})); + + uint8_t* arr = (uint8_t*)ptr; + ptr += length; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case SHORT_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT16, + tensorflow::TensorShape({length})); + + int16_t* arr = (int16_t*)ptr; + ptr += length * 2; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case INT_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT32, + tensorflow::TensorShape({length})); + + int32_t* arr = (int32_t*)ptr; + ptr += length * 4; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case LONG_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT64, + tensorflow::TensorShape({length})); + + int64_t* arr = (int64_t*)ptr; + ptr += length * 8; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case FLOAT_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_FLOAT, + tensorflow::TensorShape({length})); + + float* arr = (float*)ptr; + ptr += 4 * length; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case DOUBLE_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_DOUBLE, + tensorflow::TensorShape({length})); + + double* arr = (double*)ptr; + ptr += 8 * length; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case UCHAR_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_UINT16, + tensorflow::TensorShape({length})); + + uint16_t* arr = (uint16_t*)ptr; + ptr += length * 2; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case BOOL_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_BOOL, + tensorflow::TensorShape({length})); + + bool* arr = (bool*)ptr; + ptr += length; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case STRING_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_STRING, + tensorflow::TensorShape({length})); + + for (int32_t i = 0; i < length; i++) { + int32_t str_length = *((int32_t*)ptr); + ptr += 4; + const int8_t* str = (const int8_t*)ptr; + ptr += str_length; + tensor.vec()(i) = std::string((char*)str, str_length); + } + + out_tensors.emplace_back(std::move(tensor)); + break; + } + case DATE_ARR: { + int32_t length = *((int32_t*)ptr); + ptr += 4; + tensorflow::Tensor tensor(tensorflow::cpu_allocator(), + tensorflow::DT_INT64, + tensorflow::TensorShape({length})); + int64_t* arr = (int64_t*)ptr; + ptr += length * 8; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors.emplace_back(std::move(tensor)); + break; + } + case WRAPPED_OBJ: { + int32_t byte_arr_size = *((int32_t*)ptr); + ptr += 4; + + tensorflow::Status status = Parse(ptr, out_tensors, types); + if (!status.ok()) return status; + + int32_t offset = *((int32_t*)ptr); + ptr += 4; + + break; + } + case COMPLEX_OBJ: { + uint8_t version = *ptr; + ptr += 1; + int16_t flags = *((int16_t*)ptr); // USER_TYPE = 1, HAS_SCHEMA = 2 + ptr += 2; + int32_t type_id = *((int32_t*)ptr); + ptr += 4; + int32_t hash_code = *((int32_t*)ptr); + ptr += 4; + int32_t length = *((int32_t*)ptr); + ptr += 4; + int32_t schema_id = *((int32_t*)ptr); + ptr += 4; + int32_t schema_offset = *((int32_t*)ptr); + ptr += 4; + + uint8_t* end = ptr + schema_offset - 24; + int32_t i = 0; + while (ptr < end) { + i++; + tensorflow::Status status = Parse(ptr, out_tensors, types); + if (!status.ok()) return status; + } + + ptr += (length - schema_offset); + + break; + } + default: { + return tensorflow::errors::Internal("Unknowd binary type (type id ", + (int)object_type_id, ")"); + } + } + + return tensorflow::Status::OK(); +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h new file mode 100644 index 0000000000..1e845cbc56 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h @@ -0,0 +1,54 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include "tensorflow/core/framework/dataset.h" +#include "tensorflow/core/lib/core/status.h" + +namespace ignite { + +class BinaryObjectParser { + public: + tensorflow::Status Parse(uint8_t*& ptr, + std::vector& out_tensors, + std::vector& types); +}; + +enum ObjectType { + BYTE = 1, + SHORT = 2, + INT = 3, + LONG = 4, + FLOAT = 5, + DOUBLE = 6, + UCHAR = 7, + BOOL = 8, + STRING = 9, + DATE = 11, + BYTE_ARR = 12, + SHORT_ARR = 13, + INT_ARR = 14, + LONG_ARR = 15, + FLOAT_ARR = 16, + DOUBLE_ARR = 17, + UCHAR_ARR = 18, + BOOL_ARR = 19, + STRING_ARR = 20, + DATE_ARR = 22, + WRAPPED_OBJ = 27, + COMPLEX_OBJ = 103 +}; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_client.cc b/tensorflow/contrib/ignite/kernels/ignite_client.cc new file mode 100644 index 0000000000..5a8eddb944 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_client.cc @@ -0,0 +1,55 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef IGNITE_CLIENT_H +#define IGNITE_CLIENT_H +#include "ignite_client.h" +#endif + +namespace ignite { + +tensorflow::Status Client::ReadByte(uint8_t& data) { + return ReadData((uint8_t*)&data, 1); +} + +tensorflow::Status Client::ReadShort(int16_t& data) { + return ReadData((uint8_t*)&data, 2); +} + +tensorflow::Status Client::ReadInt(int32_t& data) { + return ReadData((uint8_t*)&data, 4); +} + +tensorflow::Status Client::ReadLong(int64_t& data) { + return ReadData((uint8_t*)&data, 8); +} + +tensorflow::Status Client::WriteByte(uint8_t data) { + return WriteData((uint8_t*)&data, 1); +} + +tensorflow::Status Client::WriteShort(int16_t data) { + return WriteData((uint8_t*)&data, 2); +} + +tensorflow::Status Client::WriteInt(int32_t data) { + return WriteData((uint8_t*)&data, 4); +} + +tensorflow::Status Client::WriteLong(int64_t data) { + return WriteData((uint8_t*)&data, 8); +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_client.h b/tensorflow/contrib/ignite/kernels/ignite_client.h new file mode 100644 index 0000000000..64e28d75f0 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_client.h @@ -0,0 +1,40 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/lib/core/status.h" + +namespace ignite { + +class Client { + public: + virtual tensorflow::Status Connect() = 0; + virtual tensorflow::Status Disconnect() = 0; + virtual bool IsConnected() = 0; + virtual int GetSocketDescriptor() = 0; + + virtual tensorflow::Status ReadByte(uint8_t& data); + virtual tensorflow::Status ReadShort(int16_t& data); + virtual tensorflow::Status ReadInt(int32_t& data); + virtual tensorflow::Status ReadLong(int64_t& data); + virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length) = 0; + + virtual tensorflow::Status WriteByte(uint8_t data); + virtual tensorflow::Status WriteShort(int16_t data); + virtual tensorflow::Status WriteInt(int32_t data); + virtual tensorflow::Status WriteLong(int64_t data); + virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length) = 0; +}; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc new file mode 100644 index 0000000000..a9bf26955b --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc @@ -0,0 +1,123 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_dataset_iterator.h" +#include "tensorflow/core/platform/logging.h" + +namespace ignite { + +IgniteDataset::IgniteDataset(tensorflow::OpKernelContext* ctx, + std::string cache_name, std::string host, + tensorflow::int32 port, bool local, + tensorflow::int32 part, + tensorflow::int32 page_size, std::string username, + std::string password, std::string certfile, + std::string keyfile, std::string cert_password, + std::vector schema, + std::vector permutation) + : DatasetBase(tensorflow::DatasetContext(ctx)), + cache_name(cache_name), + host(host), + port(port), + local(local), + part(part), + page_size(page_size), + username(username), + password(password), + certfile(certfile), + keyfile(keyfile), + cert_password(cert_password), + schema(schema), + permutation(permutation) { + SchemaToTypes(); + SchemaToShapes(); + + LOG(INFO) << "Ignite Dataset created [cache_name='" << cache_name + << "', host='" << host << "', port=" << port << ", local=" << local + << ", part=" << part << ", page_size=" << page_size + << ", username='" << username << "', certfile='" << certfile + << "', keyfile='" << keyfile + "']"; +} + +IgniteDataset::~IgniteDataset() { LOG(INFO) << "Ignite Dataset destroyed"; } + +std::unique_ptr IgniteDataset::MakeIteratorInternal( + const tensorflow::string& prefix) const { + return std::unique_ptr(new IgniteDatasetIterator( + {this, tensorflow::strings::StrCat(prefix, "::Ignite")}, this->host, + this->port, this->cache_name, this->local, this->part, this->page_size, + this->username, this->password, this->certfile, this->keyfile, + this->cert_password, this->schema, this->permutation)); +} + +const tensorflow::DataTypeVector& IgniteDataset::output_dtypes() const { + return dtypes; +} + +const std::vector& +IgniteDataset::output_shapes() const { + return shapes; +} + +tensorflow::string IgniteDataset::DebugString() const { + return "IgniteDatasetOp::Dataset"; +} + +tensorflow::Status IgniteDataset::AsGraphDefInternal( + tensorflow::SerializationContext* ctx, DatasetGraphDefBuilder* b, + tensorflow::Node** output) const { + return tensorflow::errors::Unimplemented( + "IgniteDataset does not support 'AsGraphDefInternal'"); +} + +void IgniteDataset::SchemaToTypes() { + for (auto e : schema) { + if (e == BYTE || e == BYTE_ARR) { + dtypes.push_back(tensorflow::DT_UINT8); + } else if (e == SHORT || e == SHORT_ARR) { + dtypes.push_back(tensorflow::DT_INT16); + } else if (e == INT || e == INT_ARR) { + dtypes.push_back(tensorflow::DT_INT32); + } else if (e == LONG || e == LONG_ARR) { + dtypes.push_back(tensorflow::DT_INT64); + } else if (e == FLOAT || e == FLOAT_ARR) { + dtypes.push_back(tensorflow::DT_FLOAT); + } else if (e == DOUBLE || e == DOUBLE_ARR) { + dtypes.push_back(tensorflow::DT_DOUBLE); + } else if (e == UCHAR || e == UCHAR_ARR) { + dtypes.push_back(tensorflow::DT_UINT8); + } else if (e == BOOL || e == BOOL_ARR) { + dtypes.push_back(tensorflow::DT_BOOL); + } else if (e == STRING || e == STRING_ARR) { + dtypes.push_back(tensorflow::DT_STRING); + } else { + LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; + } + } +} + +void IgniteDataset::SchemaToShapes() { + for (auto e : schema) { + if (e >= 1 && e < 10) { + shapes.push_back(tensorflow::PartialTensorShape({})); + } else if (e >= 12 && e < 21) { + shapes.push_back(tensorflow::PartialTensorShape({-1})); + } else { + LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; + } + } +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.h b/tensorflow/contrib/ignite/kernels/ignite_dataset.h new file mode 100644 index 0000000000..2120dfd342 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.h @@ -0,0 +1,65 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/dataset.h" + +namespace ignite { + +class IgniteDataset : public tensorflow::DatasetBase { + public: + IgniteDataset(tensorflow::OpKernelContext* ctx, std::string cache_name, + std::string host, tensorflow::int32 port, bool local, + tensorflow::int32 part, tensorflow::int32 page_size, + std::string username, std::string password, + std::string certfile, std::string keyfile, + std::string cert_password, + std::vector schema, + std::vector permutation); + ~IgniteDataset(); + std::unique_ptr MakeIteratorInternal( + const tensorflow::string& prefix) const override; + const tensorflow::DataTypeVector& output_dtypes() const override; + const std::vector& output_shapes() + const override; + tensorflow::string DebugString() const override; + + protected: + tensorflow::Status AsGraphDefInternal( + tensorflow::SerializationContext* ctx, DatasetGraphDefBuilder* b, + tensorflow::Node** output) const override; + + private: + const std::string cache_name; + const std::string host; + const tensorflow::int32 port; + const bool local; + const tensorflow::int32 part; + const tensorflow::int32 page_size; + const std::string username; + const std::string password; + const std::string certfile; + const std::string keyfile; + const std::string cert_password; + const std::vector schema; + const std::vector permutation; + + tensorflow::DataTypeVector dtypes; + std::vector shapes; + + void SchemaToTypes(); + void SchemaToShapes(); +}; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc new file mode 100644 index 0000000000..03cc3c1291 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc @@ -0,0 +1,447 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_dataset_iterator.h" + +#include "ignite_plain_client.h" +#include "ignite_ssl_wrapper.h" +#include "tensorflow/core/platform/logging.h" + +#include +#include + +namespace ignite { + +#define CHECK_STATUS(status) \ + if (!status.ok()) return status; + +IgniteDatasetIterator::IgniteDatasetIterator( + const Params& params, std::string host, tensorflow::int32 port, + std::string cache_name, bool local, tensorflow::int32 part, + tensorflow::int32 page_size, std::string username, std::string password, + std::string certfile, std::string keyfile, std::string cert_password, + std::vector schema, + std::vector permutation) + : tensorflow::DatasetIterator(params), + cache_name(cache_name), + local(local), + part(part), + page_size(page_size), + username(username), + password(password), + schema(schema), + permutation(permutation), + remainder(-1), + cursor_id(-1), + last_page(false) { + Client* p_client = new PlainClient(host, port); + + if (certfile.empty()) + client = std::unique_ptr(p_client); + else + client = std::unique_ptr(new SslWrapper( + std::unique_ptr(p_client), certfile, keyfile, cert_password)); + + LOG(INFO) << "Ignite Dataset Iterator created"; +} + +IgniteDatasetIterator::~IgniteDatasetIterator() { + tensorflow::Status status = CloseConnection(); + if (!status.ok()) LOG(ERROR) << status.ToString(); + + LOG(INFO) << "Ignite Dataset Iterator destroyed"; +} + +tensorflow::Status IgniteDatasetIterator::EstablishConnection() { + if (!client->IsConnected()) { + tensorflow::Status status = client->Connect(); + if (!status.ok()) return status; + + status = Handshake(); + if (!status.ok()) { + tensorflow::Status disconnect_status = client->Disconnect(); + if (!disconnect_status.ok()) LOG(ERROR) << disconnect_status.ToString(); + + return status; + } + } + + return tensorflow::Status::OK(); +} + +tensorflow::Status IgniteDatasetIterator::CloseConnection() { + if (cursor_id != -1 && !last_page) { + tensorflow::Status conn_status = EstablishConnection(); + if (!conn_status.ok()) return conn_status; + + CHECK_STATUS(client->WriteInt(18)); // Message length + CHECK_STATUS( + client->WriteShort(close_connection_opcode)); // Operation code + CHECK_STATUS(client->WriteLong(0)); // Request ID + CHECK_STATUS(client->WriteLong(cursor_id)); // Resource ID + + int32_t res_len; + CHECK_STATUS(client->ReadInt(res_len)); + if (res_len < 12) + return tensorflow::errors::Internal( + "Close Resource Response is corrupted"); + + int64_t req_id; + CHECK_STATUS(client->ReadLong(req_id)); + int32_t status; + CHECK_STATUS(client->ReadInt(status)); + if (status != 0) { + uint8_t err_msg_header; + CHECK_STATUS(client->ReadByte(err_msg_header)); + if (err_msg_header == string_val) { + int32_t err_msg_length; + CHECK_STATUS(client->ReadInt(err_msg_length)); + uint8_t* err_msg_c = new uint8_t[err_msg_length]; + CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + std::string err_msg((char*)err_msg_c, err_msg_length); + delete[] err_msg_c; + + return tensorflow::errors::Internal("Close Resource Error [status=", + status, ", message=", err_msg, "]"); + } + return tensorflow::errors::Internal("Close Resource Error [status=", + status, "]"); + } + + LOG(INFO) << "Query Cursor " << cursor_id << " is closed"; + + cursor_id = -1; + + return client->Disconnect(); + } else { + LOG(INFO) << "Query Cursor " << cursor_id << " is already closed"; + } + + return client->IsConnected() ? client->Disconnect() + : tensorflow::Status::OK(); +} + +tensorflow::Status IgniteDatasetIterator::GetNextInternal( + tensorflow::IteratorContext* ctx, + std::vector* out_tensors, bool* end_of_sequence) { + if (remainder == 0 && last_page) { + LOG(INFO) << "Query Cursor " << cursor_id << " is closed"; + + cursor_id = -1; + *end_of_sequence = true; + return tensorflow::Status::OK(); + } else { + tensorflow::Status status = EstablishConnection(); + if (!status.ok()) return status; + + if (remainder == -1 || remainder == 0) { + tensorflow::Status status = + remainder == -1 ? ScanQuery() : LoadNextPage(); + if (!status.ok()) return status; + } + + uint8_t* initial_ptr = ptr; + std::vector types; + std::vector tensors; + + status = parser.Parse(ptr, tensors, types); // Parse key + if (!status.ok()) return status; + + status = parser.Parse(ptr, tensors, types); // Parse val + if (!status.ok()) return status; + + remainder -= (ptr - initial_ptr); + + out_tensors->resize(tensors.size()); + for (int32_t i = 0; i < tensors.size(); i++) + (*out_tensors)[permutation[i]] = std::move(tensors[i]); + + *end_of_sequence = false; + return tensorflow::Status::OK(); + } + + *end_of_sequence = true; + return tensorflow::Status::OK(); +} + +tensorflow::Status IgniteDatasetIterator::SaveInternal( + tensorflow::IteratorStateWriter* writer) { + return tensorflow::errors::Unimplemented( + "Iterator for IgniteDataset does not support 'SaveInternal'"); +} + +tensorflow::Status IgniteDatasetIterator::RestoreInternal( + tensorflow::IteratorContext* ctx, tensorflow::IteratorStateReader* reader) { + return tensorflow::errors::Unimplemented( + "Iterator for IgniteDataset does not support 'RestoreInternal')"); +} + +tensorflow::Status IgniteDatasetIterator::Handshake() { + int32_t msg_len = 8; + + if (username.empty()) + msg_len += 1; + else + msg_len += 5 + username.length(); + + if (password.empty()) + msg_len += 1; + else + msg_len += 5 + password.length(); + + CHECK_STATUS(client->WriteInt(msg_len)); + CHECK_STATUS(client->WriteByte(1)); + CHECK_STATUS(client->WriteShort(protocol_major_version)); + CHECK_STATUS(client->WriteShort(protocol_minor_version)); + CHECK_STATUS(client->WriteShort(protocol_patch_version)); + CHECK_STATUS(client->WriteByte(2)); + if (username.empty()) { + CHECK_STATUS(client->WriteByte(null_val)); + } else { + CHECK_STATUS(client->WriteByte(string_val)); + CHECK_STATUS(client->WriteInt(username.length())); + CHECK_STATUS( + client->WriteData((uint8_t*)username.c_str(), username.length())); + } + + if (password.empty()) { + CHECK_STATUS(client->WriteByte(null_val)); + } else { + CHECK_STATUS(client->WriteByte(string_val)); + CHECK_STATUS(client->WriteInt(password.length())); + CHECK_STATUS( + client->WriteData((uint8_t*)password.c_str(), password.length())); + } + + int32_t handshake_res_len; + CHECK_STATUS(client->ReadInt(handshake_res_len)); + uint8_t handshake_res; + CHECK_STATUS(client->ReadByte(handshake_res)); + + LOG(INFO) << "Handshake length " << handshake_res_len << ", res " + << (int16_t)handshake_res; + + if (handshake_res != 1) { + int16_t serv_ver_major; + CHECK_STATUS(client->ReadShort(serv_ver_major)); + int16_t serv_ver_minor; + CHECK_STATUS(client->ReadShort(serv_ver_minor)); + int16_t serv_ver_patch; + CHECK_STATUS(client->ReadShort(serv_ver_patch)); + uint8_t header; + CHECK_STATUS(client->ReadByte(header)); + + if (header == string_val) { + int32_t length; + CHECK_STATUS(client->ReadInt(length)); + uint8_t* err_msg_c = new uint8_t[length]; + CHECK_STATUS(client->ReadData(err_msg_c, length)); + std::string err_msg((char*)err_msg_c, length); + delete[] err_msg_c; + + return tensorflow::errors::Internal( + "Handshake Error [result=", handshake_res, ", version=", + serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, + ", message='", err_msg, "']"); + } else if (header == null_val) { + return tensorflow::errors::Internal( + "Handshake Error [result=", handshake_res, ", version=", + serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, "]"); + } else { + return tensorflow::errors::Internal( + "Handshake Error [result=", handshake_res, ", version=", + serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, "]"); + } + } + + return tensorflow::Status::OK(); +} + +tensorflow::Status IgniteDatasetIterator::ScanQuery() { + CHECK_STATUS(client->WriteInt(25)); // Message length + CHECK_STATUS(client->WriteShort(scan_query_opcode)); // Operation code + CHECK_STATUS(client->WriteLong(0)); // Request ID + CHECK_STATUS(client->WriteInt(JavaHashCode(cache_name))); // Cache name + CHECK_STATUS(client->WriteByte(0)); // Flags + CHECK_STATUS(client->WriteByte(null_val)); // Filter object + CHECK_STATUS(client->WriteInt(page_size)); // Cursor page size + CHECK_STATUS(client->WriteInt(part)); // Partition to query + CHECK_STATUS(client->WriteByte(local)); // Local flag + + int64_t wait_start = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + int32_t res_len; + CHECK_STATUS(client->ReadInt(res_len)); + + int64_t wait_stop = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + LOG(INFO) << "Scan Query waited " << (wait_stop - wait_start) << " ms"; + + if (res_len < 12) + return tensorflow::errors::Internal("Scan Query Response is corrupted"); + + int64_t req_id; + CHECK_STATUS(client->ReadLong(req_id)); + + int32_t status; + CHECK_STATUS(client->ReadInt(status)); + + if (status != 0) { + uint8_t err_msg_header; + CHECK_STATUS(client->ReadByte(err_msg_header)); + + if (err_msg_header == string_val) { + int32_t err_msg_length; + CHECK_STATUS(client->ReadInt(err_msg_length)); + + uint8_t* err_msg_c = new uint8_t[err_msg_length]; + CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + std::string err_msg((char*)err_msg_c, err_msg_length); + delete[] err_msg_c; + + return tensorflow::errors::Internal("Scan Query Error [status=", status, + ", message=", err_msg, "]"); + } + return tensorflow::errors::Internal("Scan Query Error [status=", status, + "]"); + } + + CHECK_STATUS(client->ReadLong(cursor_id)); + + LOG(INFO) << "Query Cursor " << cursor_id << " is opened"; + + int32_t row_cnt; + CHECK_STATUS(client->ReadInt(row_cnt)); + + remainder = res_len - 25; + page = std::unique_ptr(new uint8_t[remainder]); + ptr = page.get(); + + int64_t start = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + CHECK_STATUS(client->ReadData(ptr, remainder)); + + int64_t stop = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + ; + + double size_in_mb = 1.0 * remainder / 1024 / 1024; + double time_in_s = 1.0 * (stop - start) / 1000; + LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 + << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; + + uint8_t last_page_b; + CHECK_STATUS(client->ReadByte(last_page_b)); + + last_page = !last_page_b; + + return tensorflow::Status::OK(); +} + +tensorflow::Status IgniteDatasetIterator::LoadNextPage() { + CHECK_STATUS(client->WriteInt(18)); // Message length + CHECK_STATUS(client->WriteShort(load_next_page_opcode)); // Operation code + CHECK_STATUS(client->WriteLong(0)); // Request ID + CHECK_STATUS(client->WriteLong(cursor_id)); // Cursor ID + + int64_t wait_start = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + int32_t res_len; + CHECK_STATUS(client->ReadInt(res_len)); + + int64_t wait_stop = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + LOG(INFO) << "Load Next Page waited " << (wait_stop - wait_start) << " ms"; + + if (res_len < 12) + return tensorflow::errors::Internal("Load Next Page Response is corrupted"); + + int64_t req_id; + CHECK_STATUS(client->ReadLong(req_id)); + + int32_t status; + CHECK_STATUS(client->ReadInt(status)); + + if (status != 0) { + uint8_t err_msg_header; + CHECK_STATUS(client->ReadByte(err_msg_header)); + + if (err_msg_header == string_val) { + int32_t err_msg_length; + CHECK_STATUS(client->ReadInt(err_msg_length)); + + uint8_t* err_msg_c = new uint8_t[err_msg_length]; + CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + std::string err_msg((char*)err_msg_c, err_msg_length); + delete[] err_msg_c; + + return tensorflow::errors::Internal("Load Next Page Error [status=", + status, ", message=", err_msg, "]"); + } + return tensorflow::errors::Internal("Load Next Page Error [status=", status, + "]"); + } + + int32_t row_cnt; + CHECK_STATUS(client->ReadInt(row_cnt)); + + remainder = res_len - 17; + page = std::unique_ptr(new uint8_t[remainder]); + ptr = page.get(); + + int64_t start = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + CHECK_STATUS(client->ReadData(ptr, remainder)); + + int64_t stop = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + ; + + double size_in_mb = 1.0 * remainder / 1024 / 1024; + double time_in_s = 1.0 * (stop - start) / 1000; + LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 + << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; + + uint8_t last_page_b; + CHECK_STATUS(client->ReadByte(last_page_b)); + + last_page = !last_page_b; + + return tensorflow::Status::OK(); +} + +int32_t IgniteDatasetIterator::JavaHashCode(std::string str) { + int32_t h = 0; + for (char& c : str) { + h = 31 * h + c; + } + return h; +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h new file mode 100644 index 0000000000..d1df4527f9 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h @@ -0,0 +1,87 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_binary_object_parser.h" +#include "ignite_dataset.h" + +#ifndef IGNITE_CLIENT_H +#define IGNITE_CLIENT_H +#include "ignite_client.h" +#endif + +namespace ignite { + +class IgniteDatasetIterator + : public tensorflow::DatasetIterator { + public: + IgniteDatasetIterator(const Params& params, std::string host, + tensorflow::int32 port, std::string cache_name, + bool local, tensorflow::int32 part, + tensorflow::int32 page_size, std::string username, + std::string password, std::string certfile, + std::string keyfile, std::string cert_password, + std::vector schema, + std::vector permutation); + ~IgniteDatasetIterator(); + tensorflow::Status GetNextInternal( + tensorflow::IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override; + + protected: + tensorflow::Status SaveInternal( + tensorflow::IteratorStateWriter* writer) override; + tensorflow::Status RestoreInternal( + tensorflow::IteratorContext* ctx, + tensorflow::IteratorStateReader* reader) override; + + private: + std::unique_ptr client; + BinaryObjectParser parser; + + const std::string cache_name; + const bool local; + const tensorflow::int32 part; + const tensorflow::int32 page_size; + const std::string username; + const std::string password; + const std::vector schema; + const std::vector permutation; + + int32_t remainder; + int64_t cursor_id; + bool last_page; + + std::unique_ptr page; + uint8_t* ptr; + + tensorflow::Status EstablishConnection(); + tensorflow::Status CloseConnection(); + tensorflow::Status Handshake(); + tensorflow::Status ScanQuery(); + tensorflow::Status LoadNextPage(); + int32_t JavaHashCode(std::string str); +}; + +constexpr uint8_t null_val = 101; +constexpr uint8_t string_val = 9; +constexpr uint8_t protocol_major_version = 1; +constexpr uint8_t protocol_minor_version = 1; +constexpr uint8_t protocol_patch_version = 0; +constexpr int16_t scan_query_opcode = 2000; +constexpr int16_t load_next_page_opcode = 2001; +constexpr int16_t close_connection_opcode = 0; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc new file mode 100644 index 0000000000..543b5e4afc --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -0,0 +1,145 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_dataset.h" +#include +#include "tensorflow/core/framework/dataset.h" + +namespace tensorflow { + +class IgniteDatasetOp : public DatasetOpKernel { + public: + using DatasetOpKernel::DatasetOpKernel; + + void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { + std::string cache_name = ""; + std::string host = ""; + int32 port = -1; + bool local = false; + int32 part = -1; + int32 page_size = -1; + std::string username = ""; + std::string password = ""; + std::string certfile = ""; + std::string keyfile = ""; + std::string cert_password = ""; + + const char* env_cache_name = std::getenv("IGNITE_DATASET_CACHE_NAME"); + const char* env_host = std::getenv("IGNITE_DATASET_HOST"); + const char* env_port = std::getenv("IGNITE_DATASET_PORT"); + const char* env_local = std::getenv("IGNITE_DATASET_LOCAL"); + const char* env_part = std::getenv("IGNITE_DATASET_PART"); + const char* env_page_size = std::getenv("IGNITE_DATASET_PAGE_SIZE"); + const char* env_username = std::getenv("IGNITE_DATASET_USERNAME"); + const char* env_password = std::getenv("IGNITE_DATASET_PASSWORD"); + const char* env_certfile = std::getenv("IGNITE_DATASET_CERTFILE"); + const char* env_keyfile = std::getenv("IGNITE_DATASET_KEYFILE"); + const char* env_cert_password = std::getenv("IGNITE_DATASET_CERT_PASSWORD"); + + if (env_cache_name) + cache_name = std::string(env_cache_name); + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cache_name", + &cache_name)); + + if (env_host) + host = std::string(env_host); + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "host", &host)); + + if (env_port) + port = atoi(env_port); + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "port", &port)); + + if (env_local) + local = true; + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "local", &local)); + + if (env_part) + part = atoi(env_part); + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "part", &part)); + + if (env_page_size) + page_size = atoi(env_page_size); + else + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, "page_size", &page_size)); + + if (env_username) + username = std::string(env_username); + else + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "username", &username)); + + if (env_password) + password = std::string(env_password); + else + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "password", &password)); + + if (env_certfile) + certfile = std::string(env_certfile); + else + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "certfile", &certfile)); + + if (env_keyfile) + keyfile = std::string(env_keyfile); + else + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "keyfile", &keyfile)); + + if (env_cert_password) + cert_password = std::string(env_cert_password); + else + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cert_password", + &cert_password)); + + const Tensor* schema_tensor; + OP_REQUIRES_OK(ctx, ctx->input("schema", &schema_tensor)); + OP_REQUIRES(ctx, schema_tensor->dims() == 1, + errors::InvalidArgument("`schema` must be a vector.")); + + std::vector schema; + schema.reserve(schema_tensor->NumElements()); + for (int i = 0; i < schema_tensor->NumElements(); i++) { + schema.push_back(schema_tensor->flat()(i)); + } + + const Tensor* permutation_tensor; + OP_REQUIRES_OK(ctx, ctx->input("permutation", &permutation_tensor)); + OP_REQUIRES(ctx, schema_tensor->dims() == 1, + errors::InvalidArgument("`permutation` must be a vector.")); + + std::vector permutation; + permutation.reserve(permutation_tensor->NumElements()); + for (int i = 0; i < permutation_tensor->NumElements(); i++) { + permutation.push_back(permutation_tensor->flat()(i)); + } + + *output = new ignite::IgniteDataset( + ctx, cache_name, host, port, local, part, page_size, username, password, + certfile, keyfile, cert_password, std::move(schema), + std::move(permutation)); + } +}; + +REGISTER_KERNEL_BUILDER(Name("IgniteDataset").Device(DEVICE_CPU), + IgniteDatasetOp); + +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h new file mode 100644 index 0000000000..5491af68d6 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h @@ -0,0 +1,43 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef IGNITE_CLIENT_H +#define IGNITE_CLIENT_H +#include "ignite_client.h" +#endif + +#include + +namespace ignite { + +class PlainClient : public Client { + public: + PlainClient(std::string host, int port); + ~PlainClient(); + + virtual tensorflow::Status Connect(); + virtual tensorflow::Status Disconnect(); + virtual bool IsConnected(); + virtual int GetSocketDescriptor(); + virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length); + virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length); + + private: + std::string host; + int port; + int sock; +}; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc new file mode 100644 index 0000000000..dbfa4f8786 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc @@ -0,0 +1,132 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_plain_client.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/logging.h" + +namespace ignite { + +PlainClient::PlainClient(std::string host, int port) + : host(host), port(port), sock(-1) {} + +PlainClient::~PlainClient() { + if (IsConnected()) { + tensorflow::Status status = Disconnect(); + if (!status.ok()) LOG(WARNING) << status.ToString(); + } +} + +tensorflow::Status PlainClient::Connect() { + if (sock == -1) { + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock == -1) + return tensorflow::errors::Internal("Failed to create socket"); + } + + sockaddr_in server; + + server.sin_addr.s_addr = inet_addr(host.c_str()); + if (server.sin_addr.s_addr == -1) { + hostent* he; + in_addr** addr_list; + + if ((he = gethostbyname(host.c_str())) == NULL) + return tensorflow::errors::Internal("Failed to resolve hostname \"", host, + "\""); + + addr_list = (in_addr**)he->h_addr_list; + if (addr_list[0] != NULL) server.sin_addr = *addr_list[0]; + } + + server.sin_family = AF_INET; + server.sin_port = htons(port); + + if (connect(sock, (sockaddr*)&server, sizeof(server)) < 0) + return tensorflow::errors::Internal("Failed to connect to \"", host, ":", + port, "\""); + + LOG(INFO) << "Connection to \"" << host << ":" << port << "\" established"; + + return tensorflow::Status::OK(); +} + +tensorflow::Status PlainClient::Disconnect() { + int close_res = close(sock); + sock = -1; + + LOG(INFO) << "Connection to \"" << host << ":" << port << "\" is closed"; + + return close_res == 0 ? tensorflow::Status::OK() + : tensorflow::errors::Internal( + "Failed to correctly close connection"); +} + +bool PlainClient::IsConnected() { return sock != -1; } + +int PlainClient::GetSocketDescriptor() { return sock; } + +tensorflow::Status PlainClient::ReadData(uint8_t* buf, int32_t length) { + int recieved = 0; + + while (recieved < length) { + int res = recv(sock, buf, length - recieved, 0); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while reading from socket: ", res, ", ", + std::string(strerror(errno))); + + if (res == 0) + return tensorflow::errors::Internal("Server closed connection"); + + recieved += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +tensorflow::Status PlainClient::WriteData(uint8_t* buf, int32_t length) { + int sent = 0; + + while (sent < length) { + int res = send(sock, buf, length - sent, 0); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while writing into socket: ", res, ", ", + std::string(strerror(errno))); + + sent += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc new file mode 100644 index 0000000000..f78c9b3627 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -0,0 +1,143 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_plain_client.h" + +#define WIN32_LEAN_AND_MEAN +#include +#include +#include + +#pragma comment(lib, "Ws2_32.lib") +#pragma comment(lib, "Mswsock.lib") +#pragma comment(lib, "AdvApi32.lib") + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/logging.h" + +namespace ignite { + +PlainClient::PlainClient(std::string host, int port) + : host(host), port(port), sock(INVALID_SOCKET) {} + +PlainClient::~PlainClient() { + if (IsConnected()) { + tensorflow::Status status = Disconnect(); + if (!status.ok()) LOG(WARNING) << status.ToString(); + } +} + +tensorflow::Status PlainClient::Connect() { + WSADATA wsaData; + addrinfo *result = NULL, *ptr = NULL, hints; + + int res = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (res != 0) + return tensorflow::errors::Internal("WSAStartup failed with error: ", res); + + ZeroMemory(&hints, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + + res = + getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, &result); + if (res != 0) + return tensorflow::errors::Internal("Getaddrinfo failed with error: ", res); + + for (ptr = result; ptr != NULL; ptr = ptr->ai_next) { + sock = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); + if (sock == INVALID_SOCKET) { + WSACleanup(); + return tensorflow::errors::Internal("Socket failed with error: ", + WSAGetLastError()); + } + + res = connect(sock, ptr->ai_addr, (int)ptr->ai_addrlen); + if (res == SOCKET_ERROR) { + closesocket(sock); + sock = INVALID_SOCKET; + continue; + } + + break; + } + + freeaddrinfo(result); + + if (sock == INVALID_SOCKET) { + WSACleanup(); + return tensorflow::errors::Internal("Unable to connect to server"); + } + + LOG(INFO) << "Connection to \"" << host << ":" << port << "\" established"; + + return tensorflow::Status::OK(); +} + +tensorflow::Status PlainClient::Disconnect() { + int res = shutdown(sock, SD_SEND); + closesocket(sock); + WSACleanup(); + + if (res == SOCKET_ERROR) + return tensorflow::errors::Internal("Shutdown failed with error: ", + WSAGetLastError()); + else + return tensorflow::Status::OK(); +} + +bool PlainClient::IsConnected() { return sock != INVALID_SOCKET; } + +int PlainClient::GetSocketDescriptor() { return sock; } + +tensorflow::Status PlainClient::ReadData(uint8_t *buf, int32_t length) { + int recieved = 0; + + while (recieved < length) { + int res = recv(sock, buf, length - recieved, 0); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while reading from socket: ", res); + + if (res == 0) + return tensorflow::errors::Internal("Server closed connection"); + + recieved += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +tensorflow::Status PlainClient::WriteData(uint8_t *buf, int32_t length) { + int sent = 0; + + while (sent < length) { + int res = send(sock, buf, length - sent, 0); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while writing into socket: ", res); + + sent += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc new file mode 100644 index 0000000000..a1101b91f3 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc @@ -0,0 +1,149 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "ignite_ssl_wrapper.h" + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/logging.h" + +#include +#include + +namespace ignite { + +static int PasswordCb(char *buf, int size, int rwflag, void *password) { + strncpy(buf, (char *)(password), size); + buf[size - 1] = '\0'; + return (strlen(buf)); +} + +SslWrapper::SslWrapper(std::shared_ptr client, std::string certfile, + std::string keyfile, std::string cert_password) + : client(client), + certfile(certfile), + keyfile(keyfile), + cert_password(cert_password), + ctx(NULL) {} + +SslWrapper::~SslWrapper() { + if (IsConnected()) { + tensorflow::Status status = Disconnect(); + if (!status.ok()) LOG(WARNING) << status.ToString(); + } + + if (ctx != NULL) { + SSL_CTX_free(ctx); + ctx = NULL; + } +} + +tensorflow::Status SslWrapper::InitSslContext() { + OpenSSL_add_all_algorithms(); + SSL_load_error_strings(); + + ctx = SSL_CTX_new(SSLv23_method()); + if (ctx == NULL) + return tensorflow::errors::Internal("Couldn't create SSL context"); + + SSL_CTX_set_default_passwd_cb(ctx, PasswordCb); + SSL_CTX_set_default_passwd_cb_userdata(ctx, (void *)cert_password.c_str()); + + if (SSL_CTX_use_certificate_chain_file(ctx, certfile.c_str()) != 1) + return tensorflow::errors::Internal( + "Couldn't load cetificate chain (file '", certfile, "')"); + + std::string private_key_file = keyfile.empty() ? certfile : keyfile; + if (SSL_CTX_use_PrivateKey_file(ctx, private_key_file.c_str(), + SSL_FILETYPE_PEM) != 1) + return tensorflow::errors::Internal("Couldn't load private key (file '", + private_key_file, "')"); + + return tensorflow::Status::OK(); +} + +tensorflow::Status SslWrapper::Connect() { + tensorflow::Status status; + + if (ctx == NULL) { + status = InitSslContext(); + if (!status.ok()) return status; + } + + ssl = SSL_new(ctx); + if (ssl == NULL) + return tensorflow::errors::Internal("Failed to establish SSL connection"); + + status = client->Connect(); + if (!status.ok()) return status; + + SSL_set_fd(ssl, client->GetSocketDescriptor()); + if (SSL_connect(ssl) != 1) + return tensorflow::errors::Internal("Failed to establish SSL connection"); + + LOG(INFO) << "SSL connection established"; + + return tensorflow::Status::OK(); +} + +tensorflow::Status SslWrapper::Disconnect() { + SSL_free(ssl); + + LOG(INFO) << "SSL connection closed"; + + return client->Disconnect(); +} + +bool SslWrapper::IsConnected() { return client->IsConnected(); } + +int SslWrapper::GetSocketDescriptor() { return client->GetSocketDescriptor(); } + +tensorflow::Status SslWrapper::ReadData(uint8_t *buf, int32_t length) { + int recieved = 0; + + while (recieved < length) { + int res = SSL_read(ssl, buf, length - recieved); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while reading from SSL socket: ", res); + + if (res == 0) + return tensorflow::errors::Internal("Server closed SSL connection"); + + recieved += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +tensorflow::Status SslWrapper::WriteData(uint8_t *buf, int32_t length) { + int sent = 0; + + while (sent < length) { + int res = SSL_write(ssl, buf, length - sent); + + if (res < 0) + return tensorflow::errors::Internal( + "Error occured while writing into socket: ", res); + + sent += res; + buf += res; + } + + return tensorflow::Status::OK(); +} + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h new file mode 100644 index 0000000000..e0c2a242dc --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef IGNITE_CLIENT_H +#define IGNITE_CLIENT_H +#include "ignite_client.h" +#endif + +#include +#include + +namespace ignite { + +class SslWrapper : public Client { + public: + SslWrapper(std::shared_ptr client, std::string certfile, + std::string keyfile, std::string cert_password); + ~SslWrapper(); + + virtual tensorflow::Status Connect(); + virtual tensorflow::Status Disconnect(); + virtual bool IsConnected(); + virtual int GetSocketDescriptor(); + virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length); + virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length); + + private: + std::shared_ptr client; + std::string certfile; + std::string keyfile; + std::string cert_password; + SSL_CTX* ctx; + SSL* ssl; + tensorflow::Status InitSslContext(); +}; + +} // namespace ignite diff --git a/tensorflow/contrib/ignite/ops/dataset_ops.cc b/tensorflow/contrib/ignite/ops/dataset_ops.cc new file mode 100644 index 0000000000..17494d1cfd --- /dev/null +++ b/tensorflow/contrib/ignite/ops/dataset_ops.cc @@ -0,0 +1,64 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_OP("IgniteDataset") + .Input("cache_name: string") + .Input("host: string") + .Input("port: int32") + .Input("local: bool") + .Input("part: int32") + .Input("page_size: int32") + .Input("username: string") + .Input("password: string") + .Input("certfile: string") + .Input("keyfile: string") + .Input("cert_password: string") + .Input("schema: int32") + .Input("permutation: int32") + .Output("handle: variant") + .SetIsStateful() + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( +Apache Ignite is a memory-centric distributed database, caching, and processing +platform for transactional, analytical, and streaming workloads, delivering +in-memory speeds at petabyte scale. This contrib package contains an +integration between Apache Ignite and TensorFlow. The integration is based on +tf.data from TensorFlow side and Binary Client Protocol from Apache Ignite side. +It allows to use Apache Ignite as a datasource for neural network training, +inference and all other computations supported by TensorFlow. Ignite Dataset +is based on Apache Ignite Binary Client Protocol. + +cache_name: Ignite Cache Name. +host: Ignite Thin Client Host. +port: Ignite Thin Client Port. +local: Local flag that defines that data should be fetched from local host only. +part: Partition data should be fetched from. +page_size: Page size for Ignite Thin Client. +username: Username to authenticate via Ignite Thin Client. +password: Password to authenticate via Ignite Thin Client. +certfile: SSL certificate to establish SSL connection. +keyfile: Private key file to establish SSL connection. +cert_password: SSL certificate password to establish SSL connection. +schema: Internal structure that defines schema of cache objects. +permutation: Internal structure that defines permutation of cache objects. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py new file mode 100644 index 0000000000..6fa073957a --- /dev/null +++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py @@ -0,0 +1,763 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Ignite Dataset.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import socket +import struct +import ssl +import abc + +from tensorflow.contrib.ignite.python.ops import ignite_op_loader # pylint: disable=unused-import +from tensorflow.contrib.ignite.python.ops import gen_dataset_ops +from tensorflow.python.data.ops.dataset_ops import Dataset +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape + +class Readable(): + """Readable abstract class that exposes methods to do reading-related + operations. + """ + + @abc.abstractmethod + def __init__(self): + pass + + def read_byte(self): + """Reads and returnes byte.""" + return self.__read("b", 1) + + def read_short(self): + """Reads and returns short (2 bytes, little-endian).""" + return self.__read("h", 2) + + def read_int(self): + """Reads and returns int (4 bytes, little-endian).""" + return self.__read("i", 4) + + def read_long(self): + """Reads and returns long (8 bytes, little-endian).""" + return self.__read("q", 8) + + def skip(self, length): + """Skips the specified number of bytes.""" + self.read_data(length) + + @abc.abstractmethod + def read_data(self, length): + """Reads the specified number of bytes and returns them as a buffer.""" + return None + + def __read(self, data_type, length): + """Reads, unpacks and returns specified type (little-endian).""" + buffer = self.read_data(length) + return struct.unpack("<" + data_type, buffer)[0] + +class DataBuffer(Readable): + """DataBuffer class that exposes methods to read data from a byte buffer.""" + + def __init__(self, buffer): + """Constructs a new instance of DataBuffer based on the specified byte + buffer. + + Args: + buffer: Buffer to be read. + """ + Readable.__init__(self) + self.buffer = buffer + self.ptr = 0 + + def read_data(self, length): + """Reads the specified number of bytes and returns them as a buffer.""" + data_buffer = self.buffer[self.ptr:][:length] + self.ptr += length + return data_buffer + +class TcpClient(Readable): + """TcpClient class that exposes methods to read data from a socket.""" + + def __init__(self, host, port, certfile=None, keyfile=None, password=None): + """Constructs a new instance of TcpClient based on the specified host + and port. + + Args: + host: Host to be connected. + port: Port to be connected. + certfile: File in PEM format containing the certificate as well as any + number of CA certificates needed to establish the certificate’s + authenticity. + keyfile: File containing the private key (otherwise the private key + will be taken from certfile as well). + password: Password to be used if the private key is encrypted and a + password is necessary. + """ + Readable.__init__(self) + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + if certfile is not None: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(certfile, keyfile, password) + self.sock = context.wrap_socket(self.sock) + else: + if keyfile is not None: + raise Exception("SSL is disabled, keyfile must not be specified \ + (to enable SSL specify certfile)") + if password is not None: + raise Exception("SSL is disabled, password must not be specified \ + (to enable SSL specify certfile)") + + self.host = host + self.port = port + + def __enter__(self): + """Connects to host and port specified in the constructor.""" + self.sock.connect((self.host, self.port)) + return self + + def __exit__(self, t, v, traceback): + """Disconnects the socket.""" + self.sock.close() + + def write_byte(self, v): + """Writes the specified byte.""" + self.__write(v, "b") + + def write_short(self, v): + """Writes the specified short (2 bytes, little-endian).""" + self.__write(v, "h") + + def write_int(self, v): + """Writes the specified short (4 bytes, little-endian).""" + self.__write(v, "i") + + def write_long(self, v): + """Writes the specified int (8 bytes, little-endian).""" + self.__write(v, "q") + + def write_string(self, v): + """Writes the specified string.""" + self.sock.sendall(v.encode("UTF-8")) + + def read_data(self, length): + """Reads the specified number of bytes and returns them as a buffer.""" + data_buffer = None + rem = length + while rem > 0: + buf = self.sock.recv(rem) + rem = rem - len(buf) + if data_buffer is None: + data_buffer = buf + else: + data_buffer += buf + return data_buffer + + def __write(self, value, data_type): + """Packs and writes data using the specified type (little-endian).""" + data_buffer = struct.pack("<" + data_type, value) + self.sock.sendall(data_buffer) + +class BinaryType(): + """BinaryType class that encapsulated type id, type name and fields.""" + + def __init__(self, type_id, type_name, fields): + """Constructs a new instance of BinaryType.""" + self.type_id = type_id + self.type_name = type_name + self.fields = fields + +class BinaryField(): + """BinaryField class that encapsulated field name, type id and field id.""" + + def __init__(self, field_name, type_id, field_id): + """Constructs a new instance of BinaryField.""" + self.field_name = field_name + self.type_id = type_id + self.field_id = field_id + +# Binary types defined in Apache Ignite Thin client and supported by +# TensorFlow on Apache Ignite, see +# https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. +types = { + 1: (dtypes.uint8, False), + 2: (dtypes.int16, False), + 3: (dtypes.int32, False), + 4: (dtypes.int64, False), + 5: (dtypes.float32, False), + 6: (dtypes.float64, False), + 7: (dtypes.uint16, False), + 8: (dtypes.bool, False), + 9: (dtypes.string, False), + 12: (dtypes.uint8, True), + 13: (dtypes.int16, True), + 14: (dtypes.int32, True), + 15: (dtypes.int64, True), + 16: (dtypes.float32, True), + 17: (dtypes.float64, True), + 18: (dtypes.uint16, True), + 19: (dtypes.bool, True), + 20: (dtypes.string, True) +} + +class TypeTreeNode(): + """TypeTreeNode class exposes methods to format object tree structure + data. + """ + def __init__(self, name, type_id, fields=None, permutation=None): + """Constructs a new instance of TypeTreeNode. + + Args: + name: Name of the object tree node. + type_id: Type id of the object tree node. + fields: List of fields (children of the object tree node). + permutation: Permutation that should be applied to order object children. + """ + self.name = name + self.type_id = type_id + self.fields = fields + self.permutation = permutation + + def to_output_classes(self): + """Formats the tree object the way required in 'output_classes' property of + dataset. + """ + if self.fields is None: + return ops.Tensor + output_classes = {} + for field in self.fields: + output_classes[field.name] = field.to_output_classes() + return output_classes + + def to_output_shapes(self): + """Formats the tree object the way required in 'output_shapes' property of + dataset. + """ + if self.fields is None: + object_type = types[self.type_id] + if object_type is not None: + is_array = object_type[1] + if is_array: + return tensor_shape.TensorShape([None]) + return tensor_shape.TensorShape([]) + raise Exception("Unsupported type [type_id=%d]" % self.type_id) + output_shapes = {} + for field in self.fields: + output_shapes[field.name] = field.to_output_shapes() + return output_shapes + + def to_output_types(self): + """Formats the tree object the way required in 'output_types' property of + dataset. + """ + if self.fields is None: + object_type = types[self.type_id] + if object_type is not None: + return object_type[0] + raise Exception("Unsupported type [type_id=%d]" % self.type_id) + else: + output_types = {} + for field in self.fields: + output_types[field.name] = field.to_output_types() + return output_types + + def to_flat(self): + """Returns a list of leaf node types.""" + return self.to_flat_rec([]) + + def to_permutation(self): + """Returns a permutation that should be applied to order object leafs.""" + correct_order_dict = {} + self.traversal_rec(correct_order_dict, 0) + object_order = [] + self.traversal_permutation_rec(object_order) + return [correct_order_dict[o] for o in object_order] + + def to_flat_rec(self, flat): + """Formats a list of leaf node types.""" + flat.append(self.type_id) + if self.fields is not None: + for field in self.fields: + field.to_flat_rec(flat) + return flat + + def traversal_permutation_rec(self, permutation): + """Collects nodes in accordance with permutation.""" + if self.fields is None: + permutation.append(self) + else: + for idx in self.permutation: + field = self.fields[idx] + field.traversal_permutation_rec(permutation) + + def traversal_rec(self, d, i): + """Collects nodes in pre-order traversal.""" + if self.fields is None: + d[self] = i + i += 1 + else: + for field in self.fields: + i = field.traversal_rec(d, i) + return i + +class IgniteClient(TcpClient): + """IgniteClient class exposes methods to work with Apache Ignite using Thin + client. This client works with assumption that all object in the cache + have the same structure (homogeneous objects) and the cache contains at + least one object. + """ + def __init__(self, host, port, username=None, password=None, certfile=None,\ + keyfile=None, cert_password=None): + """Constructs a new instance of IgniteClient. + + Args: + host: Apache Ignite Thin client host to be connected. + port: Apache Ignite Thin client port to be connected. + username: Apache Ignite Thin Client authentication username. + password: Apache Ignite Thin Client authentication password. + certfile: File in PEM format containing the certificate as well as + any number of CA certificates needed to establish the certificate’s + authenticity. + keyfile: File containing the private key (otherwise the private key + will be taken from certfile as well). + cert_password: Password to be used if the private key is encrypted and a + password is necessary. + """ + TcpClient.__init__(self, host, port, certfile, keyfile, cert_password) + self.username = username + self.password = password + + def handshake(self): + """Makes a handshake required to be made after connect before any other + calls. + """ + msg_len = 8 + + if self.username is None: + msg_len += 1 + else: + msg_len += 5 + len(self.username) + + if self.password is None: + msg_len += 1 + else: + msg_len += 5 + len(self.password) + + self.write_int(msg_len) # Message length + self.write_byte(1) # Handshake operation + self.write_short(1) # Version (1.1.0) + self.write_short(1) + self.write_short(0) + self.write_byte(2) # Thin client + + if self.username is None: # Username + self.write_byte(101) + else: + self.write_byte(9) + self.write_int(len(self.username)) + self.write_string(self.username) + + if self.password is None: # Password + self.write_byte(101) + else: + self.write_byte(9) + self.write_int(len(self.password)) + self.write_string(self.password) + + self.read_int() # Result length + res = self.read_byte() + + if res != 1: + serv_ver_major = self.read_short() + serv_ver_minor = self.read_short() + serv_ver_patch = self.read_short() + err_msg = self.__parse_string() + if err_msg is None: + raise Exception("Handshake Error [result=%d, version=%d.%d.%d]" \ + % (res, serv_ver_major, serv_ver_minor, serv_ver_patch)) + else: + raise Exception("Handshake Error [result=%d, version=%d.%d.%d, \ + message='%s']" % ( + res, + serv_ver_major, + serv_ver_minor, + serv_ver_patch, + err_msg + )) + + def get_cache_type(self, cache_name): + """Collects type information about objects stored in the specified + cache. + """ + cache_name_hash = self.__java_hash_code(cache_name) + self.write_int(25) # Message length + self.write_short(2000) # Operation code + self.write_long(0) # Request ID + self.write_int(cache_name_hash) # Cache name + self.write_byte(0) # Flags + self.write_byte(101) # Filter (NULL) + self.write_int(1) # Cursor page size + self.write_int(-1) # Partition to query + self.write_byte(0) # Local flag + + result_length = self.read_int() + self.read_long() # Request id + status = self.read_int() + + if status != 0: + err_msg = self.__parse_string() + if err_msg is None: + raise Exception("Scan Query Error [status=%s]" % status) + else: + raise Exception("Scan Query Error [status=%s, message='%s']" \ + % (status, err_msg)) + + self.read_long() # Cursor id + row_count = self.read_int() + + if row_count == 0: + raise Exception("Scan Query returned empty result, so it's \ + impossible to derive the cache type") + + payload = DataBuffer(self.read_data(result_length - 25)) + + self.read_byte() # Next page + + res = TypeTreeNode("root", 0, [ + self.__collect_types("key", payload), + self.__collect_types("val", payload) + ], [0, 1]) + + return res + + def __java_hash_code(self, s): + """Computes hash code of the specified string using Java code.""" + h = 0 + for c in s: + h = (31 * h + ord(c)) & 0xFFFFFFFF + return ((h + 0x80000000) & 0xFFFFFFFF) - 0x80000000 + + def __collect_types(self, field_name, data): + """Extracts type information from the specified object.""" + type_id = data.read_byte() + + # Byte scalar. + if type_id == 1: + data.skip(1) + return TypeTreeNode(field_name, type_id) + + # Short scalar. + if type_id == 2: + data.skip(2) + return TypeTreeNode(field_name, type_id) + + # Integer scalar. + if type_id == 3: + data.skip(4) + return TypeTreeNode(field_name, type_id) + + # Long scalar. + if type_id == 4: + data.skip(8) + return TypeTreeNode(field_name, type_id) + + # Float scalar. + if type_id == 5: + data.skip(4) + return TypeTreeNode(field_name, type_id) + + # Double scalar. + if type_id == 6: + data.skip(8) + return TypeTreeNode(field_name, type_id) + + # Char scalar. + if type_id == 7: + data.skip(2) + return TypeTreeNode(field_name, type_id) + + # Bool scalar. + if type_id == 8: + data.skip(1) + return TypeTreeNode(field_name, type_id) + + # String scalar. + if type_id == 9: + length = data.read_int() + data.skip(length) + return TypeTreeNode(field_name, type_id) + + # UUID scalar. + if type_id == 10: + data.skip(16) + return TypeTreeNode(field_name, type_id) + + # Date scalar. + if type_id == 11: + data.skip(8) + return TypeTreeNode(field_name, type_id) + + # Byte array. + if type_id == 12: + length = data.read_int() + data.skip(length) + return TypeTreeNode(field_name, type_id) + + # Short array. + if type_id == 13: + length = data.read_int() + data.skip(length * 2) + return TypeTreeNode(field_name, type_id) + + # Integer array. + if type_id == 14: + length = data.read_int() + data.skip(length * 4) + return TypeTreeNode(field_name, type_id) + + # Long array. + if type_id == 15: + length = data.read_int() + data.skip(length * 8) + return TypeTreeNode(field_name, type_id) + + # Float array. + if type_id == 16: + length = data.read_int() + data.skip(length * 4) + return TypeTreeNode(field_name, type_id) + + # Double array. + if type_id == 17: + length = data.read_int() + data.skip(length * 8) + return TypeTreeNode(field_name, type_id) + + # Char array. + if type_id == 18: + length = data.read_int() + data.skip(length * 2) + return TypeTreeNode(field_name, type_id) + + # Bool array. + if type_id == 19: + length = data.read_int() + data.skip(length) + return TypeTreeNode(field_name, type_id) + + # String array. + if type_id == 20: + length = data.read_int() + for _ in range(length): + header = data.read_byte() + if header == 9: + str_length = data.read_int() + data.skip(str_length) + elif header == 101: + pass + else: + raise Exception("Unknown binary type when expected string \ + [type_id=%d]" % header) + return TypeTreeNode(field_name, type_id) + + # UUID array. + if type_id == 21: + length = data.read_int() + data.skip(length * 16) # TODO: support NULL values. + return TypeTreeNode(field_name, type_id) + + # Date array. + if type_id == 22: + length = data.read_int() + data.skip(length * 8) + return TypeTreeNode(field_name, type_id) + + # Wrapped Binary Object. + if type_id == 27: + length = data.read_int() + inner_data = data.read_data(length) + data.read_int() # Offset + return self.__collect_types(field_name, DataBuffer(inner_data)) + + # Complex Object. + if type_id == 103: + data.read_byte() # Object version + data.read_short() # Object flags + obj_type_id = data.read_int() + data.read_int() # Object hash code + obj_length = data.read_int() + data.read_int() # Object schema id + obj_schema_offset = data.read_int() + + obj_type = self.__get_type(obj_type_id) + children = [] + + for obj_field in obj_type.fields: + child = self.__collect_types(obj_field.field_name, data) + children.append(child) + + children_sorted = sorted(children, key=lambda child: child.name) + permutation = [children_sorted.index(child) for child in children] + children = children_sorted + + data.skip(obj_length - obj_schema_offset) + + return TypeTreeNode(field_name, type_id, children, permutation) + + raise Exception("Unknown binary type [type_id=%d]" % type_id) + + def __get_type(self, type_id): + """Queries Apache Ignite information about type by type id.""" + self.write_int(14) # Message length + self.write_short(3002) # Operation code + self.write_long(0) # Request ID + self.write_int(type_id) # Type ID + + self.read_int() # Result length + self.read_long() # Request id + status = self.read_int() + + if status != 0: + err_msg = self.__parse_string() + if err_msg is None: + raise Exception("Get Binary Type Error [status=%d, message='%s']" \ + % (status, err_msg)) + else: + raise Exception("Get Binary Type Error [status=%d]" % status) + + binary_type_exists = self.read_byte() + + if binary_type_exists == 0: + raise Exception("Binary type not found [type_id=%d] " % type_id) + + binary_type_id = self.read_int() + binary_type_name = self.__parse_string() + self.__parse_string() # Affinity field name + + fields = [] + for _ in range(self.read_int()): + field_name = self.__parse_string() + field_type_id = self.read_int() + field_id = self.read_int() + + field = BinaryField(field_name, field_type_id, field_id) + fields.append(field) + + is_enum = self.read_byte() + if is_enum == 1: + raise Exception("Enum fields are not supported yet") + + schema_cnt = self.read_int() + for _ in range(schema_cnt): + self.read_int() # Schema id + field_cnt = self.read_int() + self.skip(field_cnt * 4) + + return BinaryType(binary_type_id, binary_type_name, fields) + + def __parse_string(self): + """Parses string.""" + header = self.read_byte() + if header == 9: + length = self.read_int() + return self.read_data(length).decode("utf-8") + if header == 101: + return None + raise Exception("Unknown binary type when expected string [type_id=%d]" \ + % header) + +class IgniteDataset(Dataset): + """Apache Ignite is a memory-centric distributed database, caching, and + processing platform for transactional, analytical, and streaming workloads, + delivering in-memory speeds at petabyte scale. This contrib package + contains an integration between Apache Ignite and TensorFlow. The + integration is based on tf.data from TensorFlow side and Binary Client + Protocol from Apache Ignite side. It allows to use Apache Ignite as a + datasource for neural network training, inference and all other + computations supported by TensorFlow. Ignite Dataset is based on Apache + Ignite Binary Client Protocol. + """ + + def __init__(self, cache_name, host="localhost", port=10800, local=False,\ + part=-1, page_size=100, username=None, password=None, certfile=None,\ + keyfile=None, cert_password=None): + """Create a IgniteDataset. + + Args: + cache_name: Cache name to be used as datasource. + host: Apache Ignite Thin Client host to be connected. + port: Apache Ignite Thin Client port to be connected. + local: Local flag that defines to query only local data. + part: Number of partitions to be queried. + page_size: Apache Ignite Thin Client page size. + username: Apache Ignite Thin Client authentication username. + password: Apache Ignite Thin Client authentication password. + certfile: File in PEM format containing the certificate as well as + any number of CA certificates needed to establish the certificate’s + authenticity. + keyfile: File containing the private key (otherwise the private key + will be taken from certfile as well). + cert_password: Password to be used if the private key is encrypted and a + password is necessary. + """ + super(IgniteDataset, self).__init__() + + with IgniteClient(host, port, username, password, certfile, keyfile,\ + cert_password) as client: + client.handshake() + self.cache_type = client.get_cache_type(cache_name) + + self.cache_name = ops.convert_to_tensor(cache_name, dtype=dtypes.string,\ + name="cache_name") + self.host = ops.convert_to_tensor(host, dtype=dtypes.string, name="host") + self.port = ops.convert_to_tensor(port, dtype=dtypes.int32, name="port") + self.local = ops.convert_to_tensor(local, dtype=dtypes.bool, name="local") + self.part = ops.convert_to_tensor(part, dtype=dtypes.int32, name="part") + self.page_size = ops.convert_to_tensor(page_size, dtype=dtypes.int32,\ + name="page_size") + self.username = ops.convert_to_tensor("" if username is None else username,\ + dtype=dtypes.string, name="username") + self.password = ops.convert_to_tensor("" if password is None else password,\ + dtype=dtypes.string, name="password") + self.certfile = ops.convert_to_tensor("" if certfile is None else certfile,\ + dtype=dtypes.string, name="certfile") + self.keyfile = ops.convert_to_tensor("" if keyfile is None else keyfile,\ + dtype=dtypes.string, name="keyfile") + self.cert_password = ops.convert_to_tensor("" if cert_password is None\ + else cert_password, dtype=dtypes.string, name="cert_password") + self.schema = ops.convert_to_tensor(self.cache_type.to_flat(),\ + dtype=dtypes.int32, name="schema") + self.permutation = ops.convert_to_tensor(self.cache_type.to_permutation(),\ + dtype=dtypes.int32, name="permutation") + + def _as_variant_tensor(self): + return gen_dataset_ops.ignite_dataset(self.cache_name, self.host,\ + self.port, self.local, self.part, self.page_size, self.username,\ + self.password, self.certfile, self.keyfile, self.cert_password,\ + self.schema, self.permutation) + + @property + def output_classes(self): + return self.cache_type.to_output_classes() + + @property + def output_shapes(self): + return self.cache_type.to_output_shapes() + + @property + def output_types(self): + return self.cache_type.to_output_types() diff --git a/tensorflow/contrib/ignite/python/ops/ignite_op_loader.py b/tensorflow/contrib/ignite/python/ops/ignite_op_loader.py new file mode 100644 index 0000000000..8115bda85b --- /dev/null +++ b/tensorflow/contrib/ignite/python/ops/ignite_op_loader.py @@ -0,0 +1,25 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Python helper for loading Ignite ops and kernels.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.util import loader +from tensorflow.python.platform import resource_loader + +_dataset_ops = loader.load_op_library( + resource_loader.get_path_to_datafile("../../_dataset_ops.so")) diff --git a/tensorflow/contrib/ignite/python/tests/bin/start-plain.sh b/tensorflow/contrib/ignite/python/tests/bin/start-plain.sh new file mode 100755 index 0000000000..f4607ce8ad --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/bin/start-plain.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +nohup apache-ignite-fabric/bin/ignite.sh /data/config/ignite-config-plain.xml & +sleep 5 # Wait Apache Ignite to be started + +./apache-ignite-fabric/bin/sqlline.sh \ +-u "jdbc:ignite:thin://127.0.0.1/" \ +--run=/data/sql/init.sql + +tail -f nohup.out diff --git a/tensorflow/contrib/ignite/python/tests/bin/start-ssl-auth.sh b/tensorflow/contrib/ignite/python/tests/bin/start-ssl-auth.sh new file mode 100755 index 0000000000..dde1162816 --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/bin/start-ssl-auth.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +nohup apache-ignite-fabric/bin/ignite.sh /data/config/ignite-config-ssl-auth.xml & +sleep 5 # Wait Apache Ignite to be started + +./apache-ignite-fabric/bin/sqlline.sh -u "jdbc:ignite:thin://127.0.0.1/?\ +sslMode=require&\ +sslClientCertificateKeyStoreUrl=/data/keystore/client.jks&\ +sslClientCertificateKeyStorePassword=123456&\ +sslTrustAll=true&\ +username=ignite&\ +password=ignite" --run=/data/sql/init.sql + +tail -f nohup.out diff --git a/tensorflow/contrib/ignite/python/tests/bin/start-ssl.sh b/tensorflow/contrib/ignite/python/tests/bin/start-ssl.sh new file mode 100755 index 0000000000..58b40b2738 --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/bin/start-ssl.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +nohup apache-ignite-fabric/bin/ignite.sh /data/config/ignite-config-ssl.xml & +sleep 5 # Wait Apache Ignite to be started + +./apache-ignite-fabric/bin/sqlline.sh -u "jdbc:ignite:thin://127.0.0.1/?\ +sslMode=require&\ +sslClientCertificateKeyStoreUrl=/data/keystore/client.jks&\ +sslClientCertificateKeyStorePassword=123456&\ +sslTrustAll=true" --run=/data/sql/init.sql --verbose=true + +tail -f nohup.out diff --git a/tensorflow/contrib/ignite/python/tests/config/ignite-config-plain.xml b/tensorflow/contrib/ignite/python/tests/config/ignite-config-plain.xml new file mode 100644 index 0000000000..d900174a8a --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/config/ignite-config-plain.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + 127.0.0.1 + + + + + + + + + diff --git a/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl-auth.xml b/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl-auth.xml new file mode 100644 index 0000000000..8e001b28ab --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl-auth.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 127.0.0.1 + + + + + + + + + diff --git a/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl.xml b/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl.xml new file mode 100644 index 0000000000..42d480c114 --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/config/ignite-config-ssl.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 127.0.0.1 + + + + + + + + + diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py new file mode 100644 index 0000000000..933e62b804 --- /dev/null +++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py @@ -0,0 +1,77 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# ============================================================================== +"""Tests for IgniteDataset.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tensorflow as tf +from tensorflow.contrib.ignite import IgniteDataset +from tensorflow.python.framework import errors +from tensorflow.python.platform import test + +class IgniteDatasetTest(test.TestCase): + """The Apache Ignite servers have to setup before the test and tear down + after the test manually. The docker engine has to be installed. + + To setup Apache Ignite servers: + $ bash start_ignite.sh + + To tear down Apache Ignite servers: + $ bash stop_ignite.sh + """ + + def test_ignite_dataset_with_plain_client(self): + ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42300) + self.__check_dataset(ds) + + def test_ignite_dataset_with_ssl_client(self): + ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42301,\ + certfile=os.path.dirname(os.path.realpath(__file__)) +\ + "/keystore/client.pem", cert_password="123456") + self.__check_dataset(ds) + + def test_ignite_dataset_with_ssl_client_and_auth(self): + ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42302,\ + certfile=os.path.dirname(os.path.realpath(__file__)) +\ + "/keystore/client.pem", cert_password="123456",\ + username="ignite", password="ignite") + self.__check_dataset(ds) + + def __check_dataset(self, dataset): + """Checks that dataset provids correct data. + """ + self.assertEquals(tf.int64, dataset.output_types['key']) + self.assertEquals(tf.string, dataset.output_types['val']['NAME']) + self.assertEquals(tf.int64, dataset.output_types['val']['VAL']) + + it = dataset.make_one_shot_iterator() + ne = it.get_next() + + with tf.Session() as sess: + rows = [sess.run(ne), sess.run(ne), sess.run(ne)] + with self.assertRaises(errors.OutOfRangeError): + sess.run(ne) + + self.assertEquals({'key': 1, 'val': {'NAME': b'TEST1', 'VAL': 42}},\ + rows[0]) + self.assertEquals({'key': 2, 'val': {'NAME': b'TEST2', 'VAL': 43}},\ + rows[1]) + self.assertEquals({'key': 3, 'val': {'NAME': b'TEST3', 'VAL': 44}},\ + rows[2]) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/ignite/python/tests/keystore/client.jks b/tensorflow/contrib/ignite/python/tests/keystore/client.jks new file mode 100644 index 0000000000000000000000000000000000000000..1875c71b605253603eb63e446da8f07cd84f64a0 GIT binary patch literal 3232 zcmezO_TO6u1_mZ5W@KPX&dE&8D`8+@G{5kwMTdcbX^%k@(+&eZHZE;8MixdbCP79< zRtAYq8p|C_4X`71ir_3D+J-^_h0{`v;Jq^I7?=cj)By2Gv~;-!V; zrk^*tG^Q9QMw#R_-+6sBwsZISYYa0FeehS&sNA)_h$$$xItuMbc#jm^U z)EsGM7v48t=yRQpg+|uW%p2uNTMy;l<&ooQ-L|Fm+-LC}Ql7ti^n{IXo#DN-aQ4pg z#|pK-#~2r{+<*SvmVdLv(*kBDMkXdk6$4&2POUbNw(q=*jI68-2IYp_2Apinp)72|OhKWB zJO*4K4u>#va6l4_#}45M8t{YUxP&>}i!xK(6EpK*3b=)tT~f$B_|rn8OXqG z;$#%#Pfm15FUm|wPt45IOU}1uEr#pj7x=Nuzv6iE!us=ylaaxU{}zcI(6gN!b!PABw|X+mmI_84U%E|> z{mQW=_S;Qr`2HyF@K#y!w@;mknUR5UakW7uIJRVkS(pqM40PBy6WTl&+kQAP^0LT_ z$@&+h<^>1)D7dHQr4}WYq^2l1rxulDre!84mZTcUvN4Cs^0A1qh&cZ>+QiDUB3!Cv z*QCz@@``+0T3;B*gQS&NBn-qFM7I9ms7T4H+0xmqwdiX68%NFUt%1m4!wL$UMuzTN zKavj~N?{J}lzRGR$J_m%x8=zk+ue9hbM_;R=>eq;wo5iLG6bg=|6T5UzizMm4#|@0 zC3nU6D?6K?{L1|Lkcsc;orbHcP1{x#s@#Ya5UGy8w0-x%FE!^&Uhc18xbL`FJu7bh zEc2IVEfii)u)od6#&v3~%l?{W-yQ1i>pxA|_=W+N&Y67;nwUKenwU00(m5x?%60vB zcixWkLv!^YNbd$_rly8Qu=H*MS!M-c39-n=bx7Ci%v0sW&eC`$Zt&A6J zx2%mi{^N~-V|3b??)9@L{4(U4*Vx*;m6w(6+2y~S{15JDb~|jS)~V&%s_R*>%7H6j zac`Q_$Ni90-+0HM@g`F8#hN-BS3=Tn<3fYRdDv6%!s;cJ0~XFrNNJJ>5CIRj);LinkxSv+h!V3q#BbzJF1B6oSQ$a90?dH&#Df;JoYLpG4o4 z_vH`lZk%qiR`UI=vL~V2U#z=$!t$%x!4s7Yn#sG}#XXJ0Jr*}aeBxPNc>0L`(;ur` z7niRTx7FTo)X_L*nZ@%!-U~AR;=(>@))7-*?Qk|v-SSgPPEf}ZT!%0vCqmjp&4Hu& pj0zRrXg*^OWubk;VkqUaBW_g*^E|kHw{Q7 +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,CE61EDD98349D0C7 + +Kzl16sj8R7YUXPCEZCqCrY4LSAjiKCRFNOagEehvN9Jpswcz4JbatoFmvVvOCgBF +7kkeCaALhfM5a+46uynZ1sOOFUOn8fUFgguN3lLInWfm6vTuXDPslg0/tRNI0YqW +ujfxyzrm1/k4RX0oLzRE1jZr69VZsBmZndkz9nkz3anWKLE7X/VIFV6U/N6YNPch +BG1Fxpt/HtM9p3B5wNDSjCVaeNP1ROKe3APLRY6k+SppTuntHV5q9Ni82r1l3ahU +zf2QvocSy9MLh+bGusJGHyJJAGuwPHm6ytPwbXGHn5xe4HPIno28j9kN7EL1ZoUs +q0PhipAkFrGIM4zg6nAwVdzY5iGySDQ3fWpz2MkrKMDRftBwA3o/M321NBUW9/2X +l+XmjXcJd0dEOslGxveb6UXLL2YvYszjQXRR4dCV/40bMJL3umRhVSay0NteoXfY +82rQchm2NHKOiDfB4RpD8JJtVQeDSMXc9TH5y2Ua7FZND60JXtFpdnfCVfVZuBJm +yBafyIsXR7EQzLG4z28Dvp4fs42A3JkF+e9Aq6Y6MmYA1wsvIKKT9HKEifqKmbgG +4E9WOZn5IWi4ZJ44VAwN/uBGrLm//3OjByeB9y8vszNbyY8dQ8x5XqnF/IzIvgqc +uKA8xuLAkTFmgRGQ/lmMDR+iMhet5dCtg9Orb9tYVL55JAb/OfsCX0LTJ3Y2RmIx +CaFpkUP7KKYD+69ajnFCxvfGnGxyBkf+JeuDYIZVFklVT9SUtL9RJh26jUdvHt2A +LQerBl8UCkVbPxsxYjdawvxuBNTD6tSRykM8zwtWcvIubp+gxE7png== +-----END RSA PRIVATE KEY----- +Bag Attributes + friendlyName: 1.2.840.113549.1.9.1=#1613636c69656e7440677269646761696e2e636f6d,CN=client,OU=Dev,O=GridGain,ST=SPb,C=RU + localKeyID: 54 69 6D 65 20 31 33 33 39 32 33 39 38 35 39 34 34 36 +subject=/C=RU/ST=SPb/O=GridGain/OU=Dev/CN=client/emailAddress=client@gridgain.com +issuer=/C=RU/ST=SPb/L=SPb/O=GridGain/OU=Dev/CN=ca/emailAddress=ca@gridgain.com +-----BEGIN CERTIFICATE----- +MIIC2TCCAkKgAwIBAgIBJDANBgkqhkiG9w0BAQUFADB3MQswCQYDVQQGEwJSVTEM +MAoGA1UECBMDU1BiMQwwCgYDVQQHEwNTUGIxETAPBgNVBAoTCEdyaWRHYWluMQww +CgYDVQQLEwNEZXYxCzAJBgNVBAMTAmNhMR4wHAYJKoZIhvcNAQkBFg9jYUBncmlk +Z2Fpbi5jb20wHhcNMTIwNjA5MTEwNDE3WhcNMzIwNjA5MTEwNDE3WjBxMQswCQYD +VQQGEwJSVTEMMAoGA1UECBMDU1BiMREwDwYDVQQKEwhHcmlkR2FpbjEMMAoGA1UE +CxMDRGV2MQ8wDQYDVQQDEwZjbGllbnQxIjAgBgkqhkiG9w0BCQEWE2NsaWVudEBn +cmlkZ2Fpbi5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANIHHcYiA+CP +EBPKNZJ6mtvN4d9Yj43B5/hzs/TK3e4XImLsMhXaElYtrXQX/SDK7Zv5zdj6AkKH +QkJ9BT8Jw7wvOQx/v4Qxrl+gTgcf6gjk6DvzqMlZUwH+ohbALj2TWsy9y+0uHKal +EVrHpbYeB9TGpD+3NHwO/CG4SySk/Y4nAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJ +YIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1Ud +DgQWBBRD/TKyBQyoVxqEupLzUB8hDrSF6DAfBgNVHSMEGDAWgBS1+Ah4ZG58tImL +KqLVX+xBKbeFUTANBgkqhkiG9w0BAQUFAAOBgQCL2vhjwcJkA1OJGuXsuO2/87Zu +HMa7gc4pm+Iol1B1gD2ksQEAU2dz/adD3369H7gZdHuk3RYPeYmD5Ppp9eECDsXc +gNWrNYaqcSTYWRAUe1/St7vB9HzPdOm/eADfQaMnal6fmjfpzTgg65A/2w4GCsqt +RL98pvdAft8v5WSx7A== +-----END CERTIFICATE----- +Bag Attributes + friendlyName: 1.2.840.113549.1.9.1=#160f636140677269646761696e2e636f6d,CN=ca,OU=Dev,O=GridGain,L=SPb,ST=SPb,C=RU +subject=/C=RU/ST=SPb/L=SPb/O=GridGain/OU=Dev/CN=ca/emailAddress=ca@gridgain.com +issuer=/C=RU/ST=SPb/L=SPb/O=GridGain/OU=Dev/CN=ca/emailAddress=ca@gridgain.com +-----BEGIN CERTIFICATE----- +MIIDSTCCArKgAwIBAgIJAKmuj925215OMA0GCSqGSIb3DQEBBQUAMHcxCzAJBgNV +BAYTAlJVMQwwCgYDVQQIEwNTUGIxDDAKBgNVBAcTA1NQYjERMA8GA1UEChMIR3Jp +ZEdhaW4xDDAKBgNVBAsTA0RldjELMAkGA1UEAxMCY2ExHjAcBgkqhkiG9w0BCQEW +D2NhQGdyaWRnYWluLmNvbTAeFw0xMjA2MDkwNjU1MTJaFw0zMjA2MDQwNjU1MTJa +MHcxCzAJBgNVBAYTAlJVMQwwCgYDVQQIEwNTUGIxDDAKBgNVBAcTA1NQYjERMA8G +A1UEChMIR3JpZEdhaW4xDDAKBgNVBAsTA0RldjELMAkGA1UEAxMCY2ExHjAcBgkq +hkiG9w0BCQEWD2NhQGdyaWRnYWluLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEAtd16DCObyM63NKF/cvRcE+8cr1dc3c7mSnTEQ61WfqPJ2QqsQAB6e+5+ +q9Np1SaJyqFTTag6483ibrU+DkGPGgEXndRHtQHQPbStWsf47DBBW2bMi6+bkPox +Cp6BhYO1DQUG5tP9CQ/g32mLQLB7LH0KtS1JcKpAClCjjWZC8b8CAwEAAaOB3DCB +2TAdBgNVHQ4EFgQUtfgIeGRufLSJiyqi1V/sQSm3hVEwgakGA1UdIwSBoTCBnoAU +tfgIeGRufLSJiyqi1V/sQSm3hVGhe6R5MHcxCzAJBgNVBAYTAlJVMQwwCgYDVQQI +EwNTUGIxDDAKBgNVBAcTA1NQYjERMA8GA1UEChMIR3JpZEdhaW4xDDAKBgNVBAsT +A0RldjELMAkGA1UEAxMCY2ExHjAcBgkqhkiG9w0BCQEWD2NhQGdyaWRnYWluLmNv +bYIJAKmuj925215OMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAhrzd +qusVLHO3wtyu0o+EAFyoDv5avCBTFsQLeDDPMyfDcEO6wfxhTanfH8C7gZc0rRnv +2nbkVbfortHIOfU2wch5gClju0cXSTIXSKOAWPIMp3HLxC/l+KpFo3epFz0rsMVB +M1ymOOdRDdAcTxcTTGY7WJXquEM3ZbT5Gh4RLDk= +-----END CERTIFICATE----- diff --git a/tensorflow/contrib/ignite/python/tests/keystore/server.jks b/tensorflow/contrib/ignite/python/tests/keystore/server.jks new file mode 100644 index 0000000000000000000000000000000000000000..006ececc31118aa18ddb6e4ec27d002e5e11646c GIT binary patch literal 3230 zcmezO_TO6u1_mZLW=c+EU|=-A@M=~y18anysevT}1GBF|6SJp56Vs*z%uI|-Oq>iW z*Y)4sc{|R}fR~L^tIebBJ1-+6D=ULRxgoa!CmVAp3!5-gP^ck~0T+nFAqM zGxPM4^K%X4#CZ*k49pBH4a`hU4UMA2d5u9_6DW7!9M(7=IgA)t8JHV;84MabnHn1z zw%)DcQJ#I`+;)?N^+jJ|gx|}o50ANf?wMD~5$CmGb&F5liCa029D8bXS&zVp76_%YhGh(^HyF~ zwr7|Ba`Hd8pV{rOp<1VwYpbqj!72x?fW^IOP9OI(F*7nSE^fSI(0J28mW?@7mXAe@ zMP%y_j*67Lnk}8(T8pm6zj4&u-Wq7oxDq6<%+k2fpmAOURy7N&msAc|I5#1sNgj|> zg;`h)m>C)WBZodKDD)c{+VAF{>n`=TFvP6j`xmuGAz17PcZI=uWA(!Y z&bto&N%UQLU;e=E#_1+&CEwpFdlI_+#kz|pEWer^JW<)8nY`Ow+|x+hV{t>oC!Xbn zr;q4A{jth*arsJdTkQ=;9gSm_Sv(Kqy&&T+F6@(L9WnLQ4rlY!EkC8?1a&MK7#J8C z*osq&%2JCUd2Mzj0|V18gC?eJ27GK>+H8z0j9N^BjEt-dEKN*{JR;+brdQtCd04*1 z$YblrpVOXBDssj0qL-e! z_3tE^3-va%w|`$L;}Nj<>_c{`U$cx$gv{s4>?kxoKIdBIo;jx!9jbh5SLQifJJrzs zKg>i-)9GBrhpt-!C&V3HuCAQCd+ibRlH+zhVzLkahd3JDTd_wa_k`B9sMrWek?13{ z*dBacz4IgAeb)@M~3AX3(WI_H1kwhYD0)`sg3qynpK@nTnN(CM+8|HvZ!6J^p;z z70HDklrAs3a-g*4a!r-uh2Qu8U1U*M%*r<}VcnA4fROmk&r_YszlR%axuOEMR(yjB1N0!)447OT5hin z`e@h^I(x}V?cc3Y48HPiKl(P^C`fr6_w|pjbFf;;&fi9lD<_^(-g^I5=a%BLPyRf3 zdU0>w&2)9mhk?Jp?O3_5|F@rk_~K+{aP0xBEtu{aG%?*YXkv1L)E11YXnAnJ3JF9l zVQ6SzVg#=xETLS3LRh7Pt@$#x zU$5)brMsWC-|mr^9a-qdEyI5}#;Jbot6!hlq^dr@w9ngL^i9Q-xAje)zPeoZQTZ3^ zI{SX>N-Unl-@TrBs?fRG_WGF4H4fa*%CAD|hiZdLaGfqI%)(^AV4%asnb79J*!IJT zk(WhYOxC|3H7_{WN5MTcFSRJKBsE3BIkl)HGc7YYu_P6#T4`B)<&VueNfCvrDJrI6 z;+q!D*D*4X2Nl)IED{D{*y|Lul*8QE$guv)rbS0jOy~Khd5i0YgcQr&WH!FHLBEbS zt>f>NJd)IQh2=oPhQ#@NHyz)uGMdNwXx?Y07a!}Fce*DBynQM9JI6`FqT^^1yQE9p z&0|vfQ<(D9Zo~)i9e#GAtb#Qsng8dHFBKsVC+4kv*LYIX@L62#V&#`f_RBX&U0c1a z>(PbZn?_qjRA{c@OZB6zBIZyQ>NJaoQmbfAcG9xQoh!B}_ieqmB!8{QbKdug03J1V AGXMYp literal 0 HcmV?d00001 diff --git a/tensorflow/contrib/ignite/python/tests/keystore/trust.jks b/tensorflow/contrib/ignite/python/tests/keystore/trust.jks new file mode 100644 index 0000000000000000000000000000000000000000..a00f1251af72982ddcd42c0274fc7b16e35dbc4c GIT binary patch literal 2432 zcmezO_TO6u1_mYu1_ov@&6J$Tz`$sJ;nlo346G4)rUsS_49va;P0XGKO-!2>Ff%bS zF>x}iT-Seh=j}K@170>xtu~Lg@4SqRtgH+M<%Zk_oNUaYENsF|L7|2`23#NxhcI(+ zKoX3{4&ezJ@Pp*IggM-cGE>|WGxJ~yxP_TrQp;c_F$*&#CmPBb$iQvlWEA61PIO2w z%1lX5%*@kE&d)WF6X!KFGB7i+G%zzYH8hG6=QRd#O`zO?b6DeiYTK0M~`xo2J_N1WG&)h#}GlWUCwLsj*=y49C6 zuc~#PS{UrR!s_waM|oTA_#FGC7{%vaao@^#!FJ2qsN+B07&u0!o#|dbd%`b6u6d2E z&0Bd{*`8hg%gO)XerC7BhH9N!uC2PB1*;sm0v7kCIepyE#LURRxVZ6-LE}vWSvKZS zSw0pq7Llz#I4V-|YPNKCYc0AO|He^sduyOU<4TacGE3t^gT{FcSk)}7UQ#(=;oO9j zCV4IU}j|ej~x1}pwMq*Xxnpl)oW3m;_Zj-th?0T!Vt59?_bm&ggGBm3)7z>`CbM7waybu>5Lv@I+;UX7X-#aZe+0kHrlU zpLmuRo<5@g^v5dK#pNr-ZM8QXbu^AyX7N0b_kxVSxUf%}b;Q(HJDkl^xBQfn6V$N; z=QFnCoXpg`5=dT~Ujff+Om_{Mm~I+0F*!l<8lwuD4+boV5qZqe(7?pd9G1r{p`3Yusegj@mdB7|T4H-iv0|n$PFANDV)NGG0cU@waJ*LF`pkF}vl;>}-9-F+vnl=Q|(^p;SV?%ERZzY3?`&i;A!#xEwPb|i4%8u8Uvb$1eYh92y{F&m$t+tc7u(KqqxIcWy#+@bx7{2HmC%bh_b>gOa=@FI&7Q?Z61tmKb#nOS>(lJ{R>j_f`fe& z+*9*XixNvxQxu$2i%K%nGLsWaQjs#H^IxM)tUN2irCN4P`Wzsy$hW2Sg@HUMp)0dU z7>Hrdk!UH0xv`O<`__--gNIU>gFB_3zS;41|L1LaGRJl|p3|KDNMm|HX@l*Ojf@Pz z>BWDSJKwL{E5AdsqT1)rRfQ@yA_YXM<1cOBeeg@o z`I49WD;VxOE>_Qqn?K9^?x~m%==KUWoEfg)yv(f;8ne#OZ(yH^Yyw;UAp^O`|Tcy z*^z~Q+%o)!W1Q;OzWVi8s0iAC-TxuCwpAuEgR={N3xB zrwW~`ZLg2%T;ss~to$lf(|*h1D}QX(Ns1^`O;Iro6W_FGzK#(&X@C8fO^c46n9lQ0 z^A^_)2`QGl$!vUYgMJ-vTF2ijc_gXr3d@0n4T7ZHcaD>UMaR)3c1f4Go5!T|r!eKI-G~q3JN)cKSp{oOGXKvXUn)W#PRv{TuJNR% v;j_5f#mX;}?3Ztly0&^-*P{!+H%&8h)e~82@&Dp^^P Date: Fri, 24 Aug 2018 18:15:57 +0300 Subject: [PATCH 0045/1085] Remove duplicated header from README.md. --- tensorflow/contrib/ignite/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md index 9054344e94..f2596fc572 100644 --- a/tensorflow/contrib/ignite/README.md +++ b/tensorflow/contrib/ignite/README.md @@ -1,4 +1,3 @@ -### Ignite Dataset # Ignite Dataset - [Overview](#overview) @@ -164,4 +163,4 @@ After that you will be able to work with it following way: ## Limitations -Presently Ignite Dataset works with assumption that all objects in the cache have the same structure (homogeneous objects) and the cache contains at least one object. Another limitation concerns structured objects, Ignite Dataset does not support UUID, Maps and Object arrays that might be parts of object structures. \ No newline at end of file +Presently Ignite Dataset works with assumption that all objects in the cache have the same structure (homogeneous objects) and the cache contains at least one object. Another limitation concerns structured objects, Ignite Dataset does not support UUID, Maps and Object arrays that might be parts of object structures. -- GitLab From 241c1740ee26b57b7a5fe8f72b9d34f4515af760 Mon Sep 17 00:00:00 2001 From: dmitrievanthony Date: Sun, 26 Aug 2018 16:03:04 +0000 Subject: [PATCH 0046/1085] Update after review: change 'ignite' namespace to 'tensorflow', rename variables to satisty code style, use pointers instead of references. --- tensorflow/contrib/ignite/BUILD | 1 - tensorflow/contrib/ignite/__init__.py | 4 +- .../kernels/ignite_binary_object_parser.cc | 322 +++++++--------- .../kernels/ignite_binary_object_parser.h | 9 +- .../contrib/ignite/kernels/ignite_client.cc | 55 --- .../contrib/ignite/kernels/ignite_client.h | 45 ++- .../contrib/ignite/kernels/ignite_dataset.cc | 105 +++-- .../contrib/ignite/kernels/ignite_dataset.h | 65 ++-- .../ignite/kernels/ignite_dataset_iterator.cc | 358 +++++++++--------- .../ignite/kernels/ignite_dataset_iterator.h | 80 ++-- .../ignite/kernels/ignite_dataset_ops.cc | 10 +- .../ignite/kernels/ignite_plain_client.h | 21 +- .../kernels/ignite_plain_client_unix.cc | 78 ++-- .../kernels/ignite_plain_client_windows.cc | 77 ++-- .../ignite/kernels/ignite_ssl_wrapper.cc | 107 +++--- .../ignite/kernels/ignite_ssl_wrapper.h | 30 +- 16 files changed, 619 insertions(+), 748 deletions(-) delete mode 100644 tensorflow/contrib/ignite/kernels/ignite_client.cc diff --git a/tensorflow/contrib/ignite/BUILD b/tensorflow/contrib/ignite/BUILD index 9f6c666893..b7d40a99f7 100644 --- a/tensorflow/contrib/ignite/BUILD +++ b/tensorflow/contrib/ignite/BUILD @@ -40,7 +40,6 @@ cc_library( srcs = [ "kernels/ignite_dataset_ops.cc", "kernels/ignite_client.h", - "kernels/ignite_client.cc", "kernels/ignite_plain_client.h", "kernels/ignite_ssl_wrapper.h", "kernels/ignite_ssl_wrapper.cc", diff --git a/tensorflow/contrib/ignite/__init__.py b/tensorflow/contrib/ignite/__init__.py index 468920a557..b78829d0f4 100644 --- a/tensorflow/contrib/ignite/__init__.py +++ b/tensorflow/contrib/ignite/__init__.py @@ -30,9 +30,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.ignite.python.ops.ignite_dataset_ops \ -import IgniteDataset - +from tensorflow.contrib.ignite.python.ops.ignite_dataset_ops import IgniteDataset from tensorflow.python.util.all_util import remove_undocumented _allowed_symbols = [ diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc index bf0ef8766e..9bf4480d2d 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc @@ -15,290 +15,258 @@ limitations under the License. #include "ignite_binary_object_parser.h" -namespace ignite { +namespace tensorflow { -tensorflow::Status BinaryObjectParser::Parse( - uint8_t*& ptr, std::vector& out_tensors, - std::vector& types) { - uint8_t object_type_id = *ptr; - ptr += 1; +Status BinaryObjectParser::Parse(uint8_t** ptr, + std::vector* out_tensors, + std::vector* types) { + uint8_t object_type_id = **ptr; + *ptr += 1; switch (object_type_id) { case BYTE: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_UINT8, {}); - tensor.scalar()() = *((uint8_t*)ptr); - ptr += 1; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_UINT8, {}); + tensor.scalar()() = *((uint8_t*)*ptr); + *ptr += 1; + out_tensors->push_back(std::move(tensor)); break; } case SHORT: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT16, {}); - tensor.scalar()() = *((int16_t*)ptr); - ptr += 2; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_INT16, {}); + tensor.scalar()() = *((int16_t*)*ptr); + *ptr += 2; + out_tensors->push_back(std::move(tensor)); break; } case INT: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT32, {}); - tensor.scalar()() = *((int32_t*)ptr); - ptr += 4; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_INT32, {}); + tensor.scalar()() = *((int32_t*)*ptr); + *ptr += 4; + out_tensors->push_back(std::move(tensor)); break; } case LONG: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT64, {}); - tensor.scalar()() = *((int64_t*)ptr); - ptr += 8; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_INT64, {}); + tensor.scalar()() = *((int64_t*)*ptr); + *ptr += 8; + out_tensors->push_back(std::move(tensor)); break; } case FLOAT: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_FLOAT, {}); - tensor.scalar()() = *((float*)ptr); - ptr += 4; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_FLOAT, {}); + tensor.scalar()() = *((float*)*ptr); + *ptr += 4; + out_tensors->push_back(std::move(tensor)); break; } case DOUBLE: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_DOUBLE, {}); - tensor.scalar()() = *((double*)ptr); - ptr += 8; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_DOUBLE, {}); + tensor.scalar()() = *((double*)*ptr); + *ptr += 8; + out_tensors->push_back(std::move(tensor)); break; } case UCHAR: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_UINT16, {}); - tensor.scalar()() = *((uint16_t*)ptr); - ptr += 2; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_UINT16, {}); + tensor.scalar()() = *((uint16_t*)*ptr); + *ptr += 2; + out_tensors->push_back(std::move(tensor)); break; } case BOOL: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_BOOL, {}); - tensor.scalar()() = *((bool*)ptr); - ptr += 1; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_BOOL, {}); + tensor.scalar()() = *((bool*)*ptr); + *ptr += 1; + out_tensors->push_back(std::move(tensor)); break; } case STRING: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_STRING, {}); - tensor.scalar()() = std::string((char*)ptr, length); - ptr += length; - out_tensors.emplace_back(std::move(tensor)); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_STRING, {}); + tensor.scalar()() = std::string((char*)*ptr, length); + *ptr += length; + out_tensors->push_back(std::move(tensor)); break; } case DATE: { - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT64, {}); - tensor.scalar()() = *((int64_t*)ptr); - ptr += 8; - out_tensors.emplace_back(std::move(tensor)); + Tensor tensor(cpu_allocator(), DT_INT64, {}); + tensor.scalar()() = *((int64_t*)*ptr); + *ptr += 8; + out_tensors->push_back(std::move(tensor)); break; } case BYTE_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_UINT8, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_UINT8, TensorShape({length})); - uint8_t* arr = (uint8_t*)ptr; - ptr += length; + uint8_t* arr = (uint8_t*)*ptr; + *ptr += length; - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case SHORT_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT16, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_INT16, TensorShape({length})); - int16_t* arr = (int16_t*)ptr; - ptr += length * 2; + int16_t* arr = (int16_t*)*ptr; + *ptr += length * 2; - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case INT_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT32, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_INT32, TensorShape({length})); - int32_t* arr = (int32_t*)ptr; - ptr += length * 4; + int32_t* arr = (int32_t*)*ptr; + *ptr += length * 4; - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case LONG_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT64, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_INT64, TensorShape({length})); - int64_t* arr = (int64_t*)ptr; - ptr += length * 8; + int64_t* arr = (int64_t*)*ptr; + *ptr += length * 8; - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case FLOAT_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_FLOAT, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_FLOAT, TensorShape({length})); - float* arr = (float*)ptr; - ptr += 4 * length; + float* arr = (float*)*ptr; + *ptr += 4 * length; std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + out_tensors->push_back(std::move(tensor)); break; } case DOUBLE_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_DOUBLE, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_DOUBLE, TensorShape({length})); - double* arr = (double*)ptr; - ptr += 8 * length; + double* arr = (double*)*ptr; + *ptr += 8 * length; std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + out_tensors->push_back(std::move(tensor)); break; } case UCHAR_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_UINT16, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_UINT16, TensorShape({length})); - uint16_t* arr = (uint16_t*)ptr; - ptr += length * 2; + uint16_t* arr = (uint16_t*)*ptr; + *ptr += length * 2; - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case BOOL_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_BOOL, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_BOOL, TensorShape({length})); - bool* arr = (bool*)ptr; - ptr += length; + bool* arr = (bool*)*ptr; + *ptr += length; std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + out_tensors->push_back(std::move(tensor)); break; } case STRING_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_STRING, - tensorflow::TensorShape({length})); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_STRING, TensorShape({length})); for (int32_t i = 0; i < length; i++) { - int32_t str_length = *((int32_t*)ptr); - ptr += 4; - const int8_t* str = (const int8_t*)ptr; - ptr += str_length; + int32_t str_length = *((int32_t*)*ptr); + *ptr += 4; + const int8_t* str = (const int8_t*)*ptr; + *ptr += str_length; tensor.vec()(i) = std::string((char*)str, str_length); } - out_tensors.emplace_back(std::move(tensor)); + out_tensors->push_back(std::move(tensor)); break; } case DATE_ARR: { - int32_t length = *((int32_t*)ptr); - ptr += 4; - tensorflow::Tensor tensor(tensorflow::cpu_allocator(), - tensorflow::DT_INT64, - tensorflow::TensorShape({length})); - int64_t* arr = (int64_t*)ptr; - ptr += length * 8; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors.emplace_back(std::move(tensor)); + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + Tensor tensor(cpu_allocator(), DT_INT64, TensorShape({length})); + int64_t* arr = (int64_t*)*ptr; + *ptr += length * 8; + + std::copy_n(arr, length, tensor.flat().data()); + out_tensors->push_back(std::move(tensor)); break; } case WRAPPED_OBJ: { - int32_t byte_arr_size = *((int32_t*)ptr); - ptr += 4; + int32_t byte_arr_size = *((int32_t*)*ptr); + *ptr += 4; - tensorflow::Status status = Parse(ptr, out_tensors, types); - if (!status.ok()) return status; + TF_RETURN_IF_ERROR(Parse(ptr, out_tensors, types)); - int32_t offset = *((int32_t*)ptr); - ptr += 4; + int32_t offset = *((int32_t*)*ptr); + *ptr += 4; break; } case COMPLEX_OBJ: { - uint8_t version = *ptr; - ptr += 1; - int16_t flags = *((int16_t*)ptr); // USER_TYPE = 1, HAS_SCHEMA = 2 - ptr += 2; - int32_t type_id = *((int32_t*)ptr); - ptr += 4; - int32_t hash_code = *((int32_t*)ptr); - ptr += 4; - int32_t length = *((int32_t*)ptr); - ptr += 4; - int32_t schema_id = *((int32_t*)ptr); - ptr += 4; - int32_t schema_offset = *((int32_t*)ptr); - ptr += 4; - - uint8_t* end = ptr + schema_offset - 24; + uint8_t version = **ptr; + *ptr += 1; + int16_t flags = *((int16_t*)*ptr); // USER_TYPE = 1, HAS_SCHEMA = 2 + *ptr += 2; + int32_t type_id = *((int32_t*)*ptr); + *ptr += 4; + int32_t hash_code = *((int32_t*)*ptr); + *ptr += 4; + int32_t length = *((int32_t*)*ptr); + *ptr += 4; + int32_t schema_id = *((int32_t*)*ptr); + *ptr += 4; + int32_t schema_offset = *((int32_t*)*ptr); + *ptr += 4; + + uint8_t* end = *ptr + schema_offset - 24; int32_t i = 0; - while (ptr < end) { + while (*ptr < end) { i++; - tensorflow::Status status = Parse(ptr, out_tensors, types); - if (!status.ok()) return status; + TF_RETURN_IF_ERROR(Parse(ptr, out_tensors, types)); } - ptr += (length - schema_offset); + *ptr += (length - schema_offset); break; } default: { - return tensorflow::errors::Internal("Unknowd binary type (type id ", - (int)object_type_id, ")"); + return errors::Internal("Unknowd binary type (type id ", + (int)object_type_id, ")"); } } - return tensorflow::Status::OK(); + return Status::OK(); } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h index 1e845cbc56..9accbd796f 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h @@ -17,13 +17,12 @@ limitations under the License. #include "tensorflow/core/framework/dataset.h" #include "tensorflow/core/lib/core/status.h" -namespace ignite { +namespace tensorflow { class BinaryObjectParser { public: - tensorflow::Status Parse(uint8_t*& ptr, - std::vector& out_tensors, - std::vector& types); + Status Parse(uint8_t** ptr, std::vector* out_tensors, + std::vector* types); }; enum ObjectType { @@ -51,4 +50,4 @@ enum ObjectType { COMPLEX_OBJ = 103 }; -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_client.cc b/tensorflow/contrib/ignite/kernels/ignite_client.cc deleted file mode 100644 index 5a8eddb944..0000000000 --- a/tensorflow/contrib/ignite/kernels/ignite_client.cc +++ /dev/null @@ -1,55 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef IGNITE_CLIENT_H -#define IGNITE_CLIENT_H -#include "ignite_client.h" -#endif - -namespace ignite { - -tensorflow::Status Client::ReadByte(uint8_t& data) { - return ReadData((uint8_t*)&data, 1); -} - -tensorflow::Status Client::ReadShort(int16_t& data) { - return ReadData((uint8_t*)&data, 2); -} - -tensorflow::Status Client::ReadInt(int32_t& data) { - return ReadData((uint8_t*)&data, 4); -} - -tensorflow::Status Client::ReadLong(int64_t& data) { - return ReadData((uint8_t*)&data, 8); -} - -tensorflow::Status Client::WriteByte(uint8_t data) { - return WriteData((uint8_t*)&data, 1); -} - -tensorflow::Status Client::WriteShort(int16_t data) { - return WriteData((uint8_t*)&data, 2); -} - -tensorflow::Status Client::WriteInt(int32_t data) { - return WriteData((uint8_t*)&data, 4); -} - -tensorflow::Status Client::WriteLong(int64_t data) { - return WriteData((uint8_t*)&data, 8); -} - -} // namespace ignite diff --git a/tensorflow/contrib/ignite/kernels/ignite_client.h b/tensorflow/contrib/ignite/kernels/ignite_client.h index 64e28d75f0..944b3fe184 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_client.h +++ b/tensorflow/contrib/ignite/kernels/ignite_client.h @@ -13,28 +13,43 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_CLIENT_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_CLIENT_H_ + #include "tensorflow/core/lib/core/status.h" -namespace ignite { +namespace tensorflow { class Client { public: - virtual tensorflow::Status Connect() = 0; - virtual tensorflow::Status Disconnect() = 0; + virtual Status Connect() = 0; + virtual Status Disconnect() = 0; virtual bool IsConnected() = 0; virtual int GetSocketDescriptor() = 0; + virtual Status ReadData(uint8_t* buf, int32_t length) = 0; + virtual Status WriteData(uint8_t* buf, int32_t length) = 0; + + inline Status ReadByte(uint8_t* data) { return ReadData(data, 1); } + + inline Status ReadShort(int16_t* data) { return ReadData((uint8_t*)data, 2); } + + inline Status ReadInt(int32_t* data) { return ReadData((uint8_t*)data, 4); } + + inline Status ReadLong(int64_t* data) { return ReadData((uint8_t*)data, 8); } - virtual tensorflow::Status ReadByte(uint8_t& data); - virtual tensorflow::Status ReadShort(int16_t& data); - virtual tensorflow::Status ReadInt(int32_t& data); - virtual tensorflow::Status ReadLong(int64_t& data); - virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length) = 0; - - virtual tensorflow::Status WriteByte(uint8_t data); - virtual tensorflow::Status WriteShort(int16_t data); - virtual tensorflow::Status WriteInt(int32_t data); - virtual tensorflow::Status WriteLong(int64_t data); - virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length) = 0; + inline Status WriteByte(uint8_t data) { return WriteData(&data, 1); } + + inline Status WriteShort(int16_t data) { + return WriteData((uint8_t*)&data, 2); + } + + inline Status WriteInt(int32_t data) { return WriteData((uint8_t*)&data, 4); } + + inline Status WriteLong(int64_t data) { + return WriteData((uint8_t*)&data, 8); + } }; -} // namespace ignite +} // namespace tensorflow + +#endif diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc index a9bf26955b..f25f8a5b18 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc @@ -16,31 +16,29 @@ limitations under the License. #include "ignite_dataset_iterator.h" #include "tensorflow/core/platform/logging.h" -namespace ignite { +namespace tensorflow { -IgniteDataset::IgniteDataset(tensorflow::OpKernelContext* ctx, - std::string cache_name, std::string host, - tensorflow::int32 port, bool local, - tensorflow::int32 part, - tensorflow::int32 page_size, std::string username, +IgniteDataset::IgniteDataset(OpKernelContext* ctx, std::string cache_name, + std::string host, int32 port, bool local, + int32 part, int32 page_size, std::string username, std::string password, std::string certfile, std::string keyfile, std::string cert_password, - std::vector schema, - std::vector permutation) - : DatasetBase(tensorflow::DatasetContext(ctx)), - cache_name(cache_name), - host(host), - port(port), - local(local), - part(part), - page_size(page_size), - username(username), - password(password), - certfile(certfile), - keyfile(keyfile), - cert_password(cert_password), - schema(schema), - permutation(permutation) { + std::vector schema, + std::vector permutation) + : DatasetBase(DatasetContext(ctx)), + cache_name_(cache_name), + host_(host), + port_(port), + local_(local), + part_(part), + page_size_(page_size), + username_(username), + password_(password), + certfile_(certfile), + keyfile_(keyfile), + cert_password_(cert_password), + schema_(schema), + permutation_(permutation) { SchemaToTypes(); SchemaToShapes(); @@ -53,55 +51,50 @@ IgniteDataset::IgniteDataset(tensorflow::OpKernelContext* ctx, IgniteDataset::~IgniteDataset() { LOG(INFO) << "Ignite Dataset destroyed"; } -std::unique_ptr IgniteDataset::MakeIteratorInternal( - const tensorflow::string& prefix) const { - return std::unique_ptr(new IgniteDatasetIterator( - {this, tensorflow::strings::StrCat(prefix, "::Ignite")}, this->host, - this->port, this->cache_name, this->local, this->part, this->page_size, - this->username, this->password, this->certfile, this->keyfile, - this->cert_password, this->schema, this->permutation)); +std::unique_ptr IgniteDataset::MakeIteratorInternal( + const string& prefix) const { + return std::unique_ptr(new IgniteDatasetIterator( + {this, strings::StrCat(prefix, "::Ignite")}, this->host_, this->port_, + this->cache_name_, this->local_, this->part_, this->page_size_, + this->username_, this->password_, this->certfile_, this->keyfile_, + this->cert_password_, this->schema_, this->permutation_)); } -const tensorflow::DataTypeVector& IgniteDataset::output_dtypes() const { - return dtypes; -} +const DataTypeVector& IgniteDataset::output_dtypes() const { return dtypes_; } -const std::vector& -IgniteDataset::output_shapes() const { - return shapes; +const std::vector& IgniteDataset::output_shapes() const { + return shapes_; } -tensorflow::string IgniteDataset::DebugString() const { - return "IgniteDatasetOp::Dataset"; -} +string IgniteDataset::DebugString() const { return "IgniteDatasetOp::Dataset"; } -tensorflow::Status IgniteDataset::AsGraphDefInternal( - tensorflow::SerializationContext* ctx, DatasetGraphDefBuilder* b, - tensorflow::Node** output) const { - return tensorflow::errors::Unimplemented( +Status IgniteDataset::AsGraphDefInternal(SerializationContext* ctx, + DatasetGraphDefBuilder* b, + Node** output) const { + return errors::Unimplemented( "IgniteDataset does not support 'AsGraphDefInternal'"); } void IgniteDataset::SchemaToTypes() { - for (auto e : schema) { + for (auto e : schema_) { if (e == BYTE || e == BYTE_ARR) { - dtypes.push_back(tensorflow::DT_UINT8); + dtypes_.push_back(DT_UINT8); } else if (e == SHORT || e == SHORT_ARR) { - dtypes.push_back(tensorflow::DT_INT16); + dtypes_.push_back(DT_INT16); } else if (e == INT || e == INT_ARR) { - dtypes.push_back(tensorflow::DT_INT32); + dtypes_.push_back(DT_INT32); } else if (e == LONG || e == LONG_ARR) { - dtypes.push_back(tensorflow::DT_INT64); + dtypes_.push_back(DT_INT64); } else if (e == FLOAT || e == FLOAT_ARR) { - dtypes.push_back(tensorflow::DT_FLOAT); + dtypes_.push_back(DT_FLOAT); } else if (e == DOUBLE || e == DOUBLE_ARR) { - dtypes.push_back(tensorflow::DT_DOUBLE); + dtypes_.push_back(DT_DOUBLE); } else if (e == UCHAR || e == UCHAR_ARR) { - dtypes.push_back(tensorflow::DT_UINT8); + dtypes_.push_back(DT_UINT8); } else if (e == BOOL || e == BOOL_ARR) { - dtypes.push_back(tensorflow::DT_BOOL); + dtypes_.push_back(DT_BOOL); } else if (e == STRING || e == STRING_ARR) { - dtypes.push_back(tensorflow::DT_STRING); + dtypes_.push_back(DT_STRING); } else { LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; } @@ -109,15 +102,15 @@ void IgniteDataset::SchemaToTypes() { } void IgniteDataset::SchemaToShapes() { - for (auto e : schema) { + for (auto e : schema_) { if (e >= 1 && e < 10) { - shapes.push_back(tensorflow::PartialTensorShape({})); + shapes_.push_back(PartialTensorShape({})); } else if (e >= 12 && e < 21) { - shapes.push_back(tensorflow::PartialTensorShape({-1})); + shapes_.push_back(PartialTensorShape({-1})); } else { LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; } } } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.h b/tensorflow/contrib/ignite/kernels/ignite_dataset.h index 2120dfd342..d3fec5910b 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset.h +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.h @@ -15,51 +15,48 @@ limitations under the License. #include "tensorflow/core/framework/dataset.h" -namespace ignite { +namespace tensorflow { -class IgniteDataset : public tensorflow::DatasetBase { +class IgniteDataset : public DatasetBase { public: - IgniteDataset(tensorflow::OpKernelContext* ctx, std::string cache_name, - std::string host, tensorflow::int32 port, bool local, - tensorflow::int32 part, tensorflow::int32 page_size, + IgniteDataset(OpKernelContext* ctx, std::string cache_name, std::string host, + int32 port, bool local, int32 part, int32 page_size, std::string username, std::string password, std::string certfile, std::string keyfile, - std::string cert_password, - std::vector schema, - std::vector permutation); + std::string cert_password, std::vector schema, + std::vector permutation); ~IgniteDataset(); - std::unique_ptr MakeIteratorInternal( - const tensorflow::string& prefix) const override; - const tensorflow::DataTypeVector& output_dtypes() const override; - const std::vector& output_shapes() - const override; - tensorflow::string DebugString() const override; + std::unique_ptr MakeIteratorInternal( + const string& prefix) const override; + const DataTypeVector& output_dtypes() const override; + const std::vector& output_shapes() const override; + string DebugString() const override; protected: - tensorflow::Status AsGraphDefInternal( - tensorflow::SerializationContext* ctx, DatasetGraphDefBuilder* b, - tensorflow::Node** output) const override; + Status AsGraphDefInternal(SerializationContext* ctx, + DatasetGraphDefBuilder* b, + Node** output) const override; private: - const std::string cache_name; - const std::string host; - const tensorflow::int32 port; - const bool local; - const tensorflow::int32 part; - const tensorflow::int32 page_size; - const std::string username; - const std::string password; - const std::string certfile; - const std::string keyfile; - const std::string cert_password; - const std::vector schema; - const std::vector permutation; - - tensorflow::DataTypeVector dtypes; - std::vector shapes; + const std::string cache_name_; + const std::string host_; + const int32 port_; + const bool local_; + const int32 part_; + const int32 page_size_; + const std::string username_; + const std::string password_; + const std::string certfile_; + const std::string keyfile_; + const std::string cert_password_; + const std::vector schema_; + const std::vector permutation_; + + DataTypeVector dtypes_; + std::vector shapes_; void SchemaToTypes(); void SchemaToShapes(); }; -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc index 03cc3c1291..1774585ecd 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc @@ -22,270 +22,262 @@ limitations under the License. #include #include -namespace ignite { - -#define CHECK_STATUS(status) \ - if (!status.ok()) return status; +namespace tensorflow { IgniteDatasetIterator::IgniteDatasetIterator( - const Params& params, std::string host, tensorflow::int32 port, - std::string cache_name, bool local, tensorflow::int32 part, - tensorflow::int32 page_size, std::string username, std::string password, - std::string certfile, std::string keyfile, std::string cert_password, - std::vector schema, - std::vector permutation) - : tensorflow::DatasetIterator(params), - cache_name(cache_name), - local(local), - part(part), - page_size(page_size), - username(username), - password(password), - schema(schema), - permutation(permutation), - remainder(-1), - cursor_id(-1), - last_page(false) { + const Params& params, std::string host, int32 port, std::string cache_name, + bool local, int32 part, int32 page_size, std::string username, + std::string password, std::string certfile, std::string keyfile, + std::string cert_password, std::vector schema, + std::vector permutation) + : DatasetIterator(params), + cache_name_(cache_name), + local_(local), + part_(part), + page_size_(page_size), + username_(username), + password_(password), + schema_(schema), + permutation_(permutation), + remainder_(-1), + cursor_id_(-1), + last_page_(false) { Client* p_client = new PlainClient(host, port); if (certfile.empty()) - client = std::unique_ptr(p_client); + client_ = std::unique_ptr(p_client); else - client = std::unique_ptr(new SslWrapper( + client_ = std::unique_ptr(new SslWrapper( std::unique_ptr(p_client), certfile, keyfile, cert_password)); LOG(INFO) << "Ignite Dataset Iterator created"; } IgniteDatasetIterator::~IgniteDatasetIterator() { - tensorflow::Status status = CloseConnection(); + Status status = CloseConnection(); if (!status.ok()) LOG(ERROR) << status.ToString(); LOG(INFO) << "Ignite Dataset Iterator destroyed"; } -tensorflow::Status IgniteDatasetIterator::EstablishConnection() { - if (!client->IsConnected()) { - tensorflow::Status status = client->Connect(); +Status IgniteDatasetIterator::EstablishConnection() { + if (!client_->IsConnected()) { + Status status = client_->Connect(); if (!status.ok()) return status; status = Handshake(); if (!status.ok()) { - tensorflow::Status disconnect_status = client->Disconnect(); + Status disconnect_status = client_->Disconnect(); if (!disconnect_status.ok()) LOG(ERROR) << disconnect_status.ToString(); return status; } } - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status IgniteDatasetIterator::CloseConnection() { - if (cursor_id != -1 && !last_page) { - tensorflow::Status conn_status = EstablishConnection(); +Status IgniteDatasetIterator::CloseConnection() { + if (cursor_id_ != -1 && !last_page_) { + Status conn_status = EstablishConnection(); if (!conn_status.ok()) return conn_status; - CHECK_STATUS(client->WriteInt(18)); // Message length - CHECK_STATUS( - client->WriteShort(close_connection_opcode)); // Operation code - CHECK_STATUS(client->WriteLong(0)); // Request ID - CHECK_STATUS(client->WriteLong(cursor_id)); // Resource ID + TF_RETURN_IF_ERROR(client_->WriteInt(18)); // Message length + TF_RETURN_IF_ERROR( + client_->WriteShort(close_connection_opcode)); // Operation code + TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID + TF_RETURN_IF_ERROR(client_->WriteLong(cursor_id_)); // Resource ID int32_t res_len; - CHECK_STATUS(client->ReadInt(res_len)); + TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); if (res_len < 12) - return tensorflow::errors::Internal( - "Close Resource Response is corrupted"); + return errors::Internal("Close Resource Response is corrupted"); int64_t req_id; - CHECK_STATUS(client->ReadLong(req_id)); + TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); int32_t status; - CHECK_STATUS(client->ReadInt(status)); + TF_RETURN_IF_ERROR(client_->ReadInt(&status)); if (status != 0) { uint8_t err_msg_header; - CHECK_STATUS(client->ReadByte(err_msg_header)); + TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); if (err_msg_header == string_val) { int32_t err_msg_length; - CHECK_STATUS(client->ReadInt(err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); uint8_t* err_msg_c = new uint8_t[err_msg_length]; - CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); std::string err_msg((char*)err_msg_c, err_msg_length); delete[] err_msg_c; - return tensorflow::errors::Internal("Close Resource Error [status=", - status, ", message=", err_msg, "]"); + return errors::Internal("Close Resource Error [status=", status, + ", message=", err_msg, "]"); } - return tensorflow::errors::Internal("Close Resource Error [status=", - status, "]"); + return errors::Internal("Close Resource Error [status=", status, "]"); } - LOG(INFO) << "Query Cursor " << cursor_id << " is closed"; + LOG(INFO) << "Query Cursor " << cursor_id_ << " is closed"; - cursor_id = -1; + cursor_id_ = -1; - return client->Disconnect(); + return client_->Disconnect(); } else { - LOG(INFO) << "Query Cursor " << cursor_id << " is already closed"; + LOG(INFO) << "Query Cursor " << cursor_id_ << " is already closed"; } - return client->IsConnected() ? client->Disconnect() - : tensorflow::Status::OK(); + return client_->IsConnected() ? client_->Disconnect() : Status::OK(); } -tensorflow::Status IgniteDatasetIterator::GetNextInternal( - tensorflow::IteratorContext* ctx, - std::vector* out_tensors, bool* end_of_sequence) { - if (remainder == 0 && last_page) { - LOG(INFO) << "Query Cursor " << cursor_id << " is closed"; +Status IgniteDatasetIterator::GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) { + if (remainder_ == 0 && last_page_) { + LOG(INFO) << "Query Cursor " << cursor_id_ << " is closed"; - cursor_id = -1; + cursor_id_ = -1; *end_of_sequence = true; - return tensorflow::Status::OK(); + return Status::OK(); } else { - tensorflow::Status status = EstablishConnection(); + Status status = EstablishConnection(); if (!status.ok()) return status; - if (remainder == -1 || remainder == 0) { - tensorflow::Status status = - remainder == -1 ? ScanQuery() : LoadNextPage(); + if (remainder_ == -1 || remainder_ == 0) { + Status status = remainder_ == -1 ? ScanQuery() : LoadNextPage(); if (!status.ok()) return status; } - uint8_t* initial_ptr = ptr; + uint8_t* initial_ptr = ptr_; std::vector types; - std::vector tensors; + std::vector tensors; - status = parser.Parse(ptr, tensors, types); // Parse key + status = parser_.Parse(&ptr_, &tensors, &types); // Parse key if (!status.ok()) return status; - status = parser.Parse(ptr, tensors, types); // Parse val + status = parser_.Parse(&ptr_, &tensors, &types); // Parse val if (!status.ok()) return status; - remainder -= (ptr - initial_ptr); + remainder_ -= (ptr_ - initial_ptr); out_tensors->resize(tensors.size()); for (int32_t i = 0; i < tensors.size(); i++) - (*out_tensors)[permutation[i]] = std::move(tensors[i]); + (*out_tensors)[permutation_[i]] = std::move(tensors[i]); *end_of_sequence = false; - return tensorflow::Status::OK(); + return Status::OK(); } *end_of_sequence = true; - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status IgniteDatasetIterator::SaveInternal( - tensorflow::IteratorStateWriter* writer) { - return tensorflow::errors::Unimplemented( +Status IgniteDatasetIterator::SaveInternal(IteratorStateWriter* writer) { + return errors::Unimplemented( "Iterator for IgniteDataset does not support 'SaveInternal'"); } -tensorflow::Status IgniteDatasetIterator::RestoreInternal( - tensorflow::IteratorContext* ctx, tensorflow::IteratorStateReader* reader) { - return tensorflow::errors::Unimplemented( +Status IgniteDatasetIterator::RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) { + return errors::Unimplemented( "Iterator for IgniteDataset does not support 'RestoreInternal')"); } -tensorflow::Status IgniteDatasetIterator::Handshake() { +Status IgniteDatasetIterator::Handshake() { int32_t msg_len = 8; - if (username.empty()) + if (username_.empty()) msg_len += 1; else - msg_len += 5 + username.length(); + msg_len += 5 + username_.length(); - if (password.empty()) + if (password_.empty()) msg_len += 1; else - msg_len += 5 + password.length(); - - CHECK_STATUS(client->WriteInt(msg_len)); - CHECK_STATUS(client->WriteByte(1)); - CHECK_STATUS(client->WriteShort(protocol_major_version)); - CHECK_STATUS(client->WriteShort(protocol_minor_version)); - CHECK_STATUS(client->WriteShort(protocol_patch_version)); - CHECK_STATUS(client->WriteByte(2)); - if (username.empty()) { - CHECK_STATUS(client->WriteByte(null_val)); + msg_len += 5 + password_.length(); + + TF_RETURN_IF_ERROR(client_->WriteInt(msg_len)); + TF_RETURN_IF_ERROR(client_->WriteByte(1)); + TF_RETURN_IF_ERROR(client_->WriteShort(protocol_major_version)); + TF_RETURN_IF_ERROR(client_->WriteShort(protocol_minor_version)); + TF_RETURN_IF_ERROR(client_->WriteShort(protocol_patch_version)); + TF_RETURN_IF_ERROR(client_->WriteByte(2)); + if (username_.empty()) { + TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); } else { - CHECK_STATUS(client->WriteByte(string_val)); - CHECK_STATUS(client->WriteInt(username.length())); - CHECK_STATUS( - client->WriteData((uint8_t*)username.c_str(), username.length())); + TF_RETURN_IF_ERROR(client_->WriteByte(string_val)); + TF_RETURN_IF_ERROR(client_->WriteInt(username_.length())); + TF_RETURN_IF_ERROR( + client_->WriteData((uint8_t*)username_.c_str(), username_.length())); } - if (password.empty()) { - CHECK_STATUS(client->WriteByte(null_val)); + if (password_.empty()) { + TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); } else { - CHECK_STATUS(client->WriteByte(string_val)); - CHECK_STATUS(client->WriteInt(password.length())); - CHECK_STATUS( - client->WriteData((uint8_t*)password.c_str(), password.length())); + TF_RETURN_IF_ERROR(client_->WriteByte(string_val)); + TF_RETURN_IF_ERROR(client_->WriteInt(password_.length())); + TF_RETURN_IF_ERROR( + client_->WriteData((uint8_t*)password_.c_str(), password_.length())); } int32_t handshake_res_len; - CHECK_STATUS(client->ReadInt(handshake_res_len)); + TF_RETURN_IF_ERROR(client_->ReadInt(&handshake_res_len)); uint8_t handshake_res; - CHECK_STATUS(client->ReadByte(handshake_res)); + TF_RETURN_IF_ERROR(client_->ReadByte(&handshake_res)); LOG(INFO) << "Handshake length " << handshake_res_len << ", res " << (int16_t)handshake_res; if (handshake_res != 1) { int16_t serv_ver_major; - CHECK_STATUS(client->ReadShort(serv_ver_major)); + TF_RETURN_IF_ERROR(client_->ReadShort(&serv_ver_major)); int16_t serv_ver_minor; - CHECK_STATUS(client->ReadShort(serv_ver_minor)); + TF_RETURN_IF_ERROR(client_->ReadShort(&serv_ver_minor)); int16_t serv_ver_patch; - CHECK_STATUS(client->ReadShort(serv_ver_patch)); + TF_RETURN_IF_ERROR(client_->ReadShort(&serv_ver_patch)); uint8_t header; - CHECK_STATUS(client->ReadByte(header)); + TF_RETURN_IF_ERROR(client_->ReadByte(&header)); if (header == string_val) { int32_t length; - CHECK_STATUS(client->ReadInt(length)); + TF_RETURN_IF_ERROR(client_->ReadInt(&length)); uint8_t* err_msg_c = new uint8_t[length]; - CHECK_STATUS(client->ReadData(err_msg_c, length)); + TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, length)); std::string err_msg((char*)err_msg_c, length); delete[] err_msg_c; - return tensorflow::errors::Internal( - "Handshake Error [result=", handshake_res, ", version=", - serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, - ", message='", err_msg, "']"); + return errors::Internal("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, ", message='", err_msg, + "']"); } else if (header == null_val) { - return tensorflow::errors::Internal( - "Handshake Error [result=", handshake_res, ", version=", - serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, "]"); + return errors::Internal("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, "]"); } else { - return tensorflow::errors::Internal( - "Handshake Error [result=", handshake_res, ", version=", - serv_ver_major, ".", serv_ver_minor, ".", serv_ver_patch, "]"); + return errors::Internal("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, "]"); } } - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status IgniteDatasetIterator::ScanQuery() { - CHECK_STATUS(client->WriteInt(25)); // Message length - CHECK_STATUS(client->WriteShort(scan_query_opcode)); // Operation code - CHECK_STATUS(client->WriteLong(0)); // Request ID - CHECK_STATUS(client->WriteInt(JavaHashCode(cache_name))); // Cache name - CHECK_STATUS(client->WriteByte(0)); // Flags - CHECK_STATUS(client->WriteByte(null_val)); // Filter object - CHECK_STATUS(client->WriteInt(page_size)); // Cursor page size - CHECK_STATUS(client->WriteInt(part)); // Partition to query - CHECK_STATUS(client->WriteByte(local)); // Local flag +Status IgniteDatasetIterator::ScanQuery() { + TF_RETURN_IF_ERROR(client_->WriteInt(25)); // Message length + TF_RETURN_IF_ERROR(client_->WriteShort(scan_query_opcode)); // Operation code + TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID + TF_RETURN_IF_ERROR( + client_->WriteInt(JavaHashCode(cache_name_))); // Cache name + TF_RETURN_IF_ERROR(client_->WriteByte(0)); // Flags + TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); // Filter object + TF_RETURN_IF_ERROR(client_->WriteInt(page_size_)); // Cursor page size + TF_RETURN_IF_ERROR(client_->WriteInt(part_)); // part_ition to query + TF_RETURN_IF_ERROR(client_->WriteByte(local_)); // local_ flag int64_t wait_start = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); int32_t res_len; - CHECK_STATUS(client->ReadInt(res_len)); + TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); int64_t wait_stop = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -293,82 +285,81 @@ tensorflow::Status IgniteDatasetIterator::ScanQuery() { LOG(INFO) << "Scan Query waited " << (wait_stop - wait_start) << " ms"; - if (res_len < 12) - return tensorflow::errors::Internal("Scan Query Response is corrupted"); + if (res_len < 12) return errors::Internal("Scan Query Response is corrupted"); int64_t req_id; - CHECK_STATUS(client->ReadLong(req_id)); + TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); int32_t status; - CHECK_STATUS(client->ReadInt(status)); + TF_RETURN_IF_ERROR(client_->ReadInt(&status)); if (status != 0) { uint8_t err_msg_header; - CHECK_STATUS(client->ReadByte(err_msg_header)); + TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); if (err_msg_header == string_val) { int32_t err_msg_length; - CHECK_STATUS(client->ReadInt(err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); uint8_t* err_msg_c = new uint8_t[err_msg_length]; - CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); std::string err_msg((char*)err_msg_c, err_msg_length); delete[] err_msg_c; - return tensorflow::errors::Internal("Scan Query Error [status=", status, - ", message=", err_msg, "]"); + return errors::Internal("Scan Query Error [status=", status, ", message=", + err_msg, "]"); } - return tensorflow::errors::Internal("Scan Query Error [status=", status, - "]"); + return errors::Internal("Scan Query Error [status=", status, "]"); } - CHECK_STATUS(client->ReadLong(cursor_id)); + TF_RETURN_IF_ERROR(client_->ReadLong(&cursor_id_)); - LOG(INFO) << "Query Cursor " << cursor_id << " is opened"; + LOG(INFO) << "Query Cursor " << cursor_id_ << " is opened"; int32_t row_cnt; - CHECK_STATUS(client->ReadInt(row_cnt)); + TF_RETURN_IF_ERROR(client_->ReadInt(&row_cnt)); - remainder = res_len - 25; - page = std::unique_ptr(new uint8_t[remainder]); - ptr = page.get(); + remainder_ = res_len - 25; + page_ = std::unique_ptr(new uint8_t[remainder_]); + ptr_ = page_.get(); int64_t start = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - CHECK_STATUS(client->ReadData(ptr, remainder)); + TF_RETURN_IF_ERROR(client_->ReadData(ptr_, remainder_)); int64_t stop = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); ; - double size_in_mb = 1.0 * remainder / 1024 / 1024; + double size_in_mb = 1.0 * remainder_ / 1024 / 1024; double time_in_s = 1.0 * (stop - start) / 1000; LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; uint8_t last_page_b; - CHECK_STATUS(client->ReadByte(last_page_b)); + TF_RETURN_IF_ERROR(client_->ReadByte(&last_page_b)); - last_page = !last_page_b; + last_page_ = !last_page_b; - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status IgniteDatasetIterator::LoadNextPage() { - CHECK_STATUS(client->WriteInt(18)); // Message length - CHECK_STATUS(client->WriteShort(load_next_page_opcode)); // Operation code - CHECK_STATUS(client->WriteLong(0)); // Request ID - CHECK_STATUS(client->WriteLong(cursor_id)); // Cursor ID +Status IgniteDatasetIterator::LoadNextPage() { + TF_RETURN_IF_ERROR(client_->WriteInt(18)); // Message length + TF_RETURN_IF_ERROR( + client_->WriteShort(load_next_page_opcode)); // Operation code + TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID + TF_RETURN_IF_ERROR(client_->WriteLong(cursor_id_)); // Cursor ID int64_t wait_start = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); int32_t res_len; - CHECK_STATUS(client->ReadInt(res_len)); + TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); int64_t wait_stop = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -377,66 +368,65 @@ tensorflow::Status IgniteDatasetIterator::LoadNextPage() { LOG(INFO) << "Load Next Page waited " << (wait_stop - wait_start) << " ms"; if (res_len < 12) - return tensorflow::errors::Internal("Load Next Page Response is corrupted"); + return errors::Internal("Load Next Page Response is corrupted"); int64_t req_id; - CHECK_STATUS(client->ReadLong(req_id)); + TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); int32_t status; - CHECK_STATUS(client->ReadInt(status)); + TF_RETURN_IF_ERROR(client_->ReadInt(&status)); if (status != 0) { uint8_t err_msg_header; - CHECK_STATUS(client->ReadByte(err_msg_header)); + TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); if (err_msg_header == string_val) { int32_t err_msg_length; - CHECK_STATUS(client->ReadInt(err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); uint8_t* err_msg_c = new uint8_t[err_msg_length]; - CHECK_STATUS(client->ReadData(err_msg_c, err_msg_length)); + TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); std::string err_msg((char*)err_msg_c, err_msg_length); delete[] err_msg_c; - return tensorflow::errors::Internal("Load Next Page Error [status=", - status, ", message=", err_msg, "]"); + return errors::Internal("Load Next Page Error [status=", status, + ", message=", err_msg, "]"); } - return tensorflow::errors::Internal("Load Next Page Error [status=", status, - "]"); + return errors::Internal("Load Next Page Error [status=", status, "]"); } int32_t row_cnt; - CHECK_STATUS(client->ReadInt(row_cnt)); + TF_RETURN_IF_ERROR(client_->ReadInt(&row_cnt)); - remainder = res_len - 17; - page = std::unique_ptr(new uint8_t[remainder]); - ptr = page.get(); + remainder_ = res_len - 17; + page_ = std::unique_ptr(new uint8_t[remainder_]); + ptr_ = page_.get(); int64_t start = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - CHECK_STATUS(client->ReadData(ptr, remainder)); + TF_RETURN_IF_ERROR(client_->ReadData(ptr_, remainder_)); int64_t stop = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); ; - double size_in_mb = 1.0 * remainder / 1024 / 1024; + double size_in_mb = 1.0 * remainder_ / 1024 / 1024; double time_in_s = 1.0 * (stop - start) / 1000; LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; uint8_t last_page_b; - CHECK_STATUS(client->ReadByte(last_page_b)); + TF_RETURN_IF_ERROR(client_->ReadByte(&last_page_b)); - last_page = !last_page_b; + last_page_ = !last_page_b; - return tensorflow::Status::OK(); + return Status::OK(); } -int32_t IgniteDatasetIterator::JavaHashCode(std::string str) { +int32_t IgniteDatasetIterator::JavaHashCode(std::string str) const { int32_t h = 0; for (char& c : str) { h = 31 * h + c; @@ -444,4 +434,4 @@ int32_t IgniteDatasetIterator::JavaHashCode(std::string str) { return h; } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h index d1df4527f9..5858dbfcb9 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h @@ -14,65 +14,55 @@ limitations under the License. ==============================================================================*/ #include "ignite_binary_object_parser.h" -#include "ignite_dataset.h" - -#ifndef IGNITE_CLIENT_H -#define IGNITE_CLIENT_H #include "ignite_client.h" -#endif +#include "ignite_dataset.h" -namespace ignite { +namespace tensorflow { -class IgniteDatasetIterator - : public tensorflow::DatasetIterator { +class IgniteDatasetIterator : public DatasetIterator { public: - IgniteDatasetIterator(const Params& params, std::string host, - tensorflow::int32 port, std::string cache_name, - bool local, tensorflow::int32 part, - tensorflow::int32 page_size, std::string username, + IgniteDatasetIterator(const Params& params, std::string host, int32 port, + std::string cache_name, bool local, int32 part, + int32 page_size, std::string username, std::string password, std::string certfile, std::string keyfile, std::string cert_password, - std::vector schema, - std::vector permutation); + std::vector schema, + std::vector permutation); ~IgniteDatasetIterator(); - tensorflow::Status GetNextInternal( - tensorflow::IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override; + Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, + bool* end_of_sequence) override; protected: - tensorflow::Status SaveInternal( - tensorflow::IteratorStateWriter* writer) override; - tensorflow::Status RestoreInternal( - tensorflow::IteratorContext* ctx, - tensorflow::IteratorStateReader* reader) override; + Status SaveInternal(IteratorStateWriter* writer) override; + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override; private: - std::unique_ptr client; - BinaryObjectParser parser; + std::unique_ptr client_; + BinaryObjectParser parser_; - const std::string cache_name; - const bool local; - const tensorflow::int32 part; - const tensorflow::int32 page_size; - const std::string username; - const std::string password; - const std::vector schema; - const std::vector permutation; + const std::string cache_name_; + const bool local_; + const int32 part_; + const int32 page_size_; + const std::string username_; + const std::string password_; + const std::vector schema_; + const std::vector permutation_; - int32_t remainder; - int64_t cursor_id; - bool last_page; + int32_t remainder_; + int64_t cursor_id_; + bool last_page_; - std::unique_ptr page; - uint8_t* ptr; + std::unique_ptr page_; + uint8_t* ptr_; - tensorflow::Status EstablishConnection(); - tensorflow::Status CloseConnection(); - tensorflow::Status Handshake(); - tensorflow::Status ScanQuery(); - tensorflow::Status LoadNextPage(); - int32_t JavaHashCode(std::string str); + Status EstablishConnection(); + Status CloseConnection(); + Status Handshake(); + Status ScanQuery(); + Status LoadNextPage(); + int32_t JavaHashCode(std::string str) const; }; constexpr uint8_t null_val = 101; @@ -84,4 +74,4 @@ constexpr int16_t scan_query_opcode = 2000; constexpr int16_t load_next_page_opcode = 2001; constexpr int16_t close_connection_opcode = 0; -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc index 543b5e4afc..89eecf9c14 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/framework/dataset.h" namespace tensorflow { +namespace { class IgniteDatasetOp : public DatasetOpKernel { public: @@ -132,14 +133,15 @@ class IgniteDatasetOp : public DatasetOpKernel { permutation.push_back(permutation_tensor->flat()(i)); } - *output = new ignite::IgniteDataset( - ctx, cache_name, host, port, local, part, page_size, username, password, - certfile, keyfile, cert_password, std::move(schema), - std::move(permutation)); + *output = + new IgniteDataset(ctx, cache_name, host, port, local, part, page_size, + username, password, certfile, keyfile, cert_password, + std::move(schema), std::move(permutation)); } }; REGISTER_KERNEL_BUILDER(Name("IgniteDataset").Device(DEVICE_CPU), IgniteDatasetOp); +} // namespace } // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h index 5491af68d6..6f417a3cb5 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h @@ -13,31 +13,28 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef IGNITE_CLIENT_H -#define IGNITE_CLIENT_H #include "ignite_client.h" -#endif #include -namespace ignite { +namespace tensorflow { class PlainClient : public Client { public: PlainClient(std::string host, int port); ~PlainClient(); - virtual tensorflow::Status Connect(); - virtual tensorflow::Status Disconnect(); + virtual Status Connect(); + virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length); - virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length); + virtual Status ReadData(uint8_t* buf, int32_t length); + virtual Status WriteData(uint8_t* buf, int32_t length); private: - std::string host; - int port; - int sock; + const std::string host_; + const int port_; + int sock_; }; -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc index dbfa4f8786..a4c58a9563 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc @@ -29,104 +29,98 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" -namespace ignite { +namespace tensorflow { PlainClient::PlainClient(std::string host, int port) - : host(host), port(port), sock(-1) {} + : host_(host), port_(port), sock_(-1) {} PlainClient::~PlainClient() { if (IsConnected()) { - tensorflow::Status status = Disconnect(); + Status status = Disconnect(); if (!status.ok()) LOG(WARNING) << status.ToString(); } } -tensorflow::Status PlainClient::Connect() { - if (sock == -1) { - sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock == -1) - return tensorflow::errors::Internal("Failed to create socket"); +Status PlainClient::Connect() { + if (sock_ == -1) { + sock_ = socket(AF_INET, SOCK_STREAM, 0); + if (sock_ == -1) return errors::Internal("Failed to create socket"); } sockaddr_in server; - server.sin_addr.s_addr = inet_addr(host.c_str()); + server.sin_addr.s_addr = inet_addr(host_.c_str()); if (server.sin_addr.s_addr == -1) { hostent* he; in_addr** addr_list; - if ((he = gethostbyname(host.c_str())) == NULL) - return tensorflow::errors::Internal("Failed to resolve hostname \"", host, - "\""); + if ((he = gethostbyname(host_.c_str())) == NULL) + return errors::Internal("Failed to resolve hostname \"", host_, "\""); addr_list = (in_addr**)he->h_addr_list; if (addr_list[0] != NULL) server.sin_addr = *addr_list[0]; } server.sin_family = AF_INET; - server.sin_port = htons(port); + server.sin_port = htons(port_); - if (connect(sock, (sockaddr*)&server, sizeof(server)) < 0) - return tensorflow::errors::Internal("Failed to connect to \"", host, ":", - port, "\""); + if (connect(sock_, (sockaddr*)&server, sizeof(server)) < 0) + return errors::Internal("Failed to connect to \"", host_, ":", port_, "\""); - LOG(INFO) << "Connection to \"" << host << ":" << port << "\" established"; + LOG(INFO) << "Connection to \"" << host_ << ":" << port_ << "\" established"; - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status PlainClient::Disconnect() { - int close_res = close(sock); - sock = -1; +Status PlainClient::Disconnect() { + int close_res = close(sock_); + sock_ = -1; - LOG(INFO) << "Connection to \"" << host << ":" << port << "\" is closed"; + LOG(INFO) << "Connection to \"" << host_ << ":" << port_ << "\" is closed"; - return close_res == 0 ? tensorflow::Status::OK() - : tensorflow::errors::Internal( - "Failed to correctly close connection"); + return close_res == 0 + ? Status::OK() + : errors::Internal("Failed to correctly close connection"); } -bool PlainClient::IsConnected() { return sock != -1; } +bool PlainClient::IsConnected() { return sock_ != -1; } -int PlainClient::GetSocketDescriptor() { return sock; } +int PlainClient::GetSocketDescriptor() { return sock_; } -tensorflow::Status PlainClient::ReadData(uint8_t* buf, int32_t length) { +Status PlainClient::ReadData(uint8_t* buf, int32_t length) { int recieved = 0; while (recieved < length) { - int res = recv(sock, buf, length - recieved, 0); + int res = recv(sock_, buf, length - recieved, 0); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while reading from socket: ", res, ", ", - std::string(strerror(errno))); + return errors::Internal("Error occured while reading from socket: ", res, + ", ", std::string(strerror(errno))); - if (res == 0) - return tensorflow::errors::Internal("Server closed connection"); + if (res == 0) return errors::Internal("Server closed connection"); recieved += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status PlainClient::WriteData(uint8_t* buf, int32_t length) { +Status PlainClient::WriteData(uint8_t* buf, int32_t length) { int sent = 0; while (sent < length) { - int res = send(sock, buf, length - sent, 0); + int res = send(sock_, buf, length - sent, 0); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while writing into socket: ", res, ", ", - std::string(strerror(errno))); + return errors::Internal("Error occured while writing into socket: ", res, + ", ", std::string(strerror(errno))); sent += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index f78c9b3627..7ba037f2d2 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -27,48 +27,45 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" -namespace ignite { +namespace tensorflow { PlainClient::PlainClient(std::string host, int port) - : host(host), port(port), sock(INVALID_SOCKET) {} + : host_(host), port_(port), sock_(INVALID_SOCKET) {} PlainClient::~PlainClient() { if (IsConnected()) { - tensorflow::Status status = Disconnect(); + Status status = Disconnect(); if (!status.ok()) LOG(WARNING) << status.ToString(); } } -tensorflow::Status PlainClient::Connect() { +Status PlainClient::Connect() { WSADATA wsaData; addrinfo *result = NULL, *ptr = NULL, hints; int res = WSAStartup(MAKEWORD(2, 2), &wsaData); - if (res != 0) - return tensorflow::errors::Internal("WSAStartup failed with error: ", res); + if (res != 0) return errors::Internal("WSAStartup failed with error: ", res); ZeroMemory(&hints, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; - res = - getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, &result); - if (res != 0) - return tensorflow::errors::Internal("Getaddrinfo failed with error: ", res); + res = getaddrinfo(host_.c_str(), std::to_string(port_).c_str(), &hints, + &result); + if (res != 0) return errors::Internal("Getaddrinfo failed with error: ", res); for (ptr = result; ptr != NULL; ptr = ptr->ai_next) { - sock = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); - if (sock == INVALID_SOCKET) { + sock_ = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); + if (sock_ == INVALID_SOCKET) { WSACleanup(); - return tensorflow::errors::Internal("Socket failed with error: ", - WSAGetLastError()); + return errors::Internal("Socket failed with error: ", WSAGetLastError()); } - res = connect(sock, ptr->ai_addr, (int)ptr->ai_addrlen); + res = connect(sock_, ptr->ai_addr, (int)ptr->ai_addrlen); if (res == SOCKET_ERROR) { - closesocket(sock); - sock = INVALID_SOCKET; + closesocket(sock_); + sock_ = INVALID_SOCKET; continue; } @@ -77,67 +74,63 @@ tensorflow::Status PlainClient::Connect() { freeaddrinfo(result); - if (sock == INVALID_SOCKET) { + if (sock_ == INVALID_SOCKET) { WSACleanup(); - return tensorflow::errors::Internal("Unable to connect to server"); + return errors::Internal("Unable to connect to server"); } - LOG(INFO) << "Connection to \"" << host << ":" << port << "\" established"; + LOG(INFO) << "Connection to \"" << host_ << ":" << port_ << "\" established"; - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status PlainClient::Disconnect() { - int res = shutdown(sock, SD_SEND); - closesocket(sock); +Status PlainClient::Disconnect() { + int res = shutdown(sock_, SD_SEND); + closesocket(sock_); WSACleanup(); if (res == SOCKET_ERROR) - return tensorflow::errors::Internal("Shutdown failed with error: ", - WSAGetLastError()); + return errors::Internal("Shutdown failed with error: ", WSAGetLastError()); else - return tensorflow::Status::OK(); + return Status::OK(); } -bool PlainClient::IsConnected() { return sock != INVALID_SOCKET; } +bool PlainClient::IsConnected() { return sock_ != INVALID_SOCKET; } -int PlainClient::GetSocketDescriptor() { return sock; } +int PlainClient::GetSocketDescriptor() { return sock_; } -tensorflow::Status PlainClient::ReadData(uint8_t *buf, int32_t length) { +Status PlainClient::ReadData(uint8_t *buf, int32_t length) { int recieved = 0; while (recieved < length) { - int res = recv(sock, buf, length - recieved, 0); + int res = recv(sock_, buf, length - recieved, 0); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while reading from socket: ", res); + return errors::Internal("Error occured while reading from socket: ", res); - if (res == 0) - return tensorflow::errors::Internal("Server closed connection"); + if (res == 0) return errors::Internal("Server closed connection"); recieved += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status PlainClient::WriteData(uint8_t *buf, int32_t length) { +Status PlainClient::WriteData(uint8_t *buf, int32_t length) { int sent = 0; while (sent < length) { - int res = send(sock, buf, length - sent, 0); + int res = send(sock_, buf, length - sent, 0); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while writing into socket: ", res); + return errors::Internal("Error occured while writing into socket: ", res); sent += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc index a1101b91f3..a2bc6b9609 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc @@ -21,7 +21,7 @@ limitations under the License. #include #include -namespace ignite { +namespace tensorflow { static int PasswordCb(char *buf, int size, int rwflag, void *password) { strncpy(buf, (char *)(password), size); @@ -31,119 +31,112 @@ static int PasswordCb(char *buf, int size, int rwflag, void *password) { SslWrapper::SslWrapper(std::shared_ptr client, std::string certfile, std::string keyfile, std::string cert_password) - : client(client), - certfile(certfile), - keyfile(keyfile), - cert_password(cert_password), - ctx(NULL) {} + : client_(client), + certfile_(certfile), + keyfile_(keyfile), + cert_password_(cert_password), + ctx_(NULL) {} SslWrapper::~SslWrapper() { if (IsConnected()) { - tensorflow::Status status = Disconnect(); + Status status = Disconnect(); if (!status.ok()) LOG(WARNING) << status.ToString(); } - if (ctx != NULL) { - SSL_CTX_free(ctx); - ctx = NULL; + if (ctx_ != NULL) { + SSL_CTX_free(ctx_); + ctx_ = NULL; } } -tensorflow::Status SslWrapper::InitSslContext() { +Status SslWrapper::InitSslContext() { OpenSSL_add_all_algorithms(); SSL_load_error_strings(); - ctx = SSL_CTX_new(SSLv23_method()); - if (ctx == NULL) - return tensorflow::errors::Internal("Couldn't create SSL context"); + ctx_ = SSL_CTX_new(SSLv23_method()); + if (ctx_ == NULL) return errors::Internal("Couldn't create SSL context"); - SSL_CTX_set_default_passwd_cb(ctx, PasswordCb); - SSL_CTX_set_default_passwd_cb_userdata(ctx, (void *)cert_password.c_str()); + SSL_CTX_set_default_passwd_cb(ctx_, PasswordCb); + SSL_CTX_set_default_passwd_cb_userdata(ctx_, (void *)cert_password_.c_str()); - if (SSL_CTX_use_certificate_chain_file(ctx, certfile.c_str()) != 1) - return tensorflow::errors::Internal( - "Couldn't load cetificate chain (file '", certfile, "')"); + if (SSL_CTX_use_certificate_chain_file(ctx_, certfile_.c_str()) != 1) + return errors::Internal("Couldn't load cetificate chain (file '", certfile_, + "')"); - std::string private_key_file = keyfile.empty() ? certfile : keyfile; - if (SSL_CTX_use_PrivateKey_file(ctx, private_key_file.c_str(), + std::string private_key_file = keyfile_.empty() ? certfile_ : keyfile_; + if (SSL_CTX_use_PrivateKey_file(ctx_, private_key_file.c_str(), SSL_FILETYPE_PEM) != 1) - return tensorflow::errors::Internal("Couldn't load private key (file '", - private_key_file, "')"); + return errors::Internal("Couldn't load private key (file '", + private_key_file, "')"); - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status SslWrapper::Connect() { - tensorflow::Status status; - - if (ctx == NULL) { - status = InitSslContext(); - if (!status.ok()) return status; +Status SslWrapper::Connect() { + if (ctx_ == NULL) { + TF_RETURN_IF_ERROR(InitSslContext()); } - ssl = SSL_new(ctx); - if (ssl == NULL) - return tensorflow::errors::Internal("Failed to establish SSL connection"); + ssl_ = SSL_new(ctx_); + if (ssl_ == NULL) + return errors::Internal("Failed to establish SSL connection"); - status = client->Connect(); - if (!status.ok()) return status; + TF_RETURN_IF_ERROR(client_->Connect()); - SSL_set_fd(ssl, client->GetSocketDescriptor()); - if (SSL_connect(ssl) != 1) - return tensorflow::errors::Internal("Failed to establish SSL connection"); + SSL_set_fd(ssl_, client_->GetSocketDescriptor()); + if (SSL_connect(ssl_) != 1) + return errors::Internal("Failed to establish SSL connection"); LOG(INFO) << "SSL connection established"; - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status SslWrapper::Disconnect() { - SSL_free(ssl); +Status SslWrapper::Disconnect() { + SSL_free(ssl_); LOG(INFO) << "SSL connection closed"; - return client->Disconnect(); + return client_->Disconnect(); } -bool SslWrapper::IsConnected() { return client->IsConnected(); } +bool SslWrapper::IsConnected() { return client_->IsConnected(); } -int SslWrapper::GetSocketDescriptor() { return client->GetSocketDescriptor(); } +int SslWrapper::GetSocketDescriptor() { return client_->GetSocketDescriptor(); } -tensorflow::Status SslWrapper::ReadData(uint8_t *buf, int32_t length) { +Status SslWrapper::ReadData(uint8_t *buf, int32_t length) { int recieved = 0; while (recieved < length) { - int res = SSL_read(ssl, buf, length - recieved); + int res = SSL_read(ssl_, buf, length - recieved); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while reading from SSL socket: ", res); + return errors::Internal("Error occured while reading from SSL socket: ", + res); - if (res == 0) - return tensorflow::errors::Internal("Server closed SSL connection"); + if (res == 0) return errors::Internal("Server closed SSL connection"); recieved += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -tensorflow::Status SslWrapper::WriteData(uint8_t *buf, int32_t length) { +Status SslWrapper::WriteData(uint8_t *buf, int32_t length) { int sent = 0; while (sent < length) { - int res = SSL_write(ssl, buf, length - sent); + int res = SSL_write(ssl_, buf, length - sent); if (res < 0) - return tensorflow::errors::Internal( - "Error occured while writing into socket: ", res); + return errors::Internal("Error occured while writing into socket: ", res); sent += res; buf += res; } - return tensorflow::Status::OK(); + return Status::OK(); } -} // namespace ignite +} // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h index e0c2a242dc..bbba6cc181 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h @@ -13,15 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef IGNITE_CLIENT_H -#define IGNITE_CLIENT_H #include "ignite_client.h" -#endif #include #include -namespace ignite { +namespace tensorflow { class SslWrapper : public Client { public: @@ -29,21 +26,22 @@ class SslWrapper : public Client { std::string keyfile, std::string cert_password); ~SslWrapper(); - virtual tensorflow::Status Connect(); - virtual tensorflow::Status Disconnect(); + virtual Status Connect(); + virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual tensorflow::Status ReadData(uint8_t* buf, int32_t length); - virtual tensorflow::Status WriteData(uint8_t* buf, int32_t length); + virtual Status ReadData(uint8_t* buf, int32_t length); + virtual Status WriteData(uint8_t* buf, int32_t length); private: - std::shared_ptr client; - std::string certfile; - std::string keyfile; - std::string cert_password; - SSL_CTX* ctx; - SSL* ssl; - tensorflow::Status InitSslContext(); + std::shared_ptr client_; + std::string certfile_; + std::string keyfile_; + std::string cert_password_; + SSL_CTX* ctx_; + SSL* ssl_; + + Status InitSslContext(); }; -} // namespace ignite +} // namespace tensorflow -- GitLab From 1408a1563e73e69f68c1eb6f34a0976c7c950ad9 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Tue, 28 Aug 2018 11:32:57 +0300 Subject: [PATCH 0047/1085] Update README.md. --- tensorflow/contrib/ignite/README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md index f2596fc572..8fec4066c4 100644 --- a/tensorflow/contrib/ignite/README.md +++ b/tensorflow/contrib/ignite/README.md @@ -13,19 +13,20 @@ ## Overview [Apache Ignite](https://ignite.apache.org/) is a memory-centric distributed database, caching, and processing platform for -transactional, analytical, and streaming workloads, delivering in-memory speeds at petabyte scale. This contrib package contains an integration between Apache Ignite and TensorFlow. The integration is based on [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) from TensorFlow side and [Binary Client Protocol](https://apacheignite.readme.io/v2.6/docs/binary-client-protocol) from Apache Ignite side. It allows to use Apache Ignite as a datasource for neural network training, inference and all other computations supported by TensorFlow. +transactional, analytical, and streaming workloads, delivering in-memory speeds at petabyte scale. This contrib package contains an integration between Apache Ignite and TensorFlow. The integration is based on [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) from TensorFlow side and [Binary Client Protocol](https://apacheignite.readme.io/v2.6/docs/binary-client-protocol) from Apache Ignite side. It allows to use Apache Ignite as a data source for neural network training, inference and all other computations supported by TensorFlow. ## Features -Ignite Dataset provides a set of features that makes it possible to use it in a wide range of cases. The most important and interesting features are described below. +Ignite Dataset provides features that that you can use in a wide range of cases. The most important and interesting features are described below. ### Distributed In-Memory Datasource -[Apache Ignite](https://ignite.apache.org/) is a distributed in-memory database, caching, and processing platform that allows to avoid limitations of hard drive and provide high reading speed and ability to store and operate with as much data as you need in distributed cluster. Using of Ignite Dataset makes it possible to utilize all these advantages. +[Apache Ignite](https://ignite.apache.org/) is a distributed in-memory database, caching, and processing platform that provides fast data access. It allows you to avoid limitations of hard drive and and store and operate with as much data as you need in distributed cluster. You can utilize +these benefits of Apache Ignite by using Ignite Dataset. Moreover, Ignite Dataset can be used for the following use-cases: - If you have a **gigabyte** of data you can keep it on a single machine on a hard drive, but you will face with hard drive speed limitations. At the same time, you can store your data in Apache Ignite on the same machine and use it as a datasource for TensorFlow and thus avoid these limitations. - If you have a **terabyte** of data you probably still can keep it on a single machine on a hard drive, but you will face with hard drive speed limitations again. At the same time, you can store your data in Apache Ignite distributed in-memory cluster and use it as a datasource for TensorFlow and thus avoid these limitations. - If you have a **petabyte** of data you can't keep it on a single machine. At the same time, you can store your data in Apache Ignite distributed in-memory cluster and use it as a datasource for TensorFlow. -It's important that Apache Ignite is not just a step of ETL pipeline between database or data warehouse and TensorFlow. Apache Ignite is a high-grade database itself. Choosing Apache Ignite and TensorFlow you are getting everything you need to work with operational or historical data and, in the same time, an ability to use this data for neural network training and inference. +Note that Apache Ignite is not just a step of ETL pipeline between a database or a data warehouse and TensorFlow. Apache Ignite is a high-grade database itself. By choosing Apache Ignite and TensorFlow you are getting everything you need to work with operational or historical data and, at the same time, an ability to use this data for neural network training and inference. ```bash $ apache-ignite-fabric/bin/ignite.sh @@ -55,7 +56,7 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL ``` ### Structured Objects -[Apache Ignite](https://ignite.apache.org/) allows to store any objects you would like to store. These objects can have any hierarchy. Ignite Dataset provides an ability to work with such objects. +[Apache Ignite](https://ignite.apache.org/) allows to store any type of objects. These objects can have any hierarchy. Ignite Dataset provides an ability to work with such objects. ```python >>> import tensorflow as tf @@ -81,7 +82,7 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL } } ``` - Neural network training and other computations require transformations that can be done as part of [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) pipeline if you use Ignite Dataset. + Neural network training and other computations require transformations that can be done as part of [tf.data](https://www.tensorflow.org/api_docs/python/tf/data) pipeline if you use Ignite Dataset. ```python >>> import tensorflow as tf @@ -99,15 +100,15 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL ### Distributed Training -TensorFlow is a machine learning framework that [natively supports](https://www.tensorflow.org/deploy/distributed) distributed neural network training, inference and other computations. The main idea behind the distributed neural network training is an ability to calculate gradients of loss functions (squares of the errors) on every partition of data (in terms of horizontal partitioning) and then sum them to get loss function gradient of the whole dataset. +TensorFlow is a machine learning framework that [natively supports](https://www.tensorflow.org/deploy/distributed) distributed neural network training, inference and other computations. The main idea behind the distributed neural network training is the ability to calculate gradients of loss functions (squares of the errors) on every partition of data (in terms of horizontal partitioning) and then sum them to get loss function gradient of the whole dataset. -Utilizing this ability we can calculate gradients on the nodes the data is stored on, reduce them and then finally update model parameters. It allows to avoid data transfers between nodes and thus to avoid network bottleneck. +Using this ability we can calculate gradients on the nodes the data is stored on, reduce them and then finally update model parameters. It allows to avoid data transfers between nodes and thus to avoid network bottlenecks. -Apache Ignite uses horizontal partitioning to store data in distributed cluster. When we create Apache Ignite cache (or table in terms of SQL) we can specify the number of partitions the data will be partitioned on. If, for example, Apache Ignite cluster consists of 10 machines and we creates cache with 10 partitions then every machine will maintain approximately one data partition. +Apache Ignite uses horizontal partitioning to store data in distributed cluster. When we create Apache Ignite cache (or table in terms of SQL), we can specify the number of partitions the data will be partitioned on. For example, if an Apache Ignite cluster consists of 10 machines and we create cache with 10 partitions, then every machine will maintain approximately one data partition. -Ignite Dataset allows to utilize these two aspects of distributed neural network training (using TensorFlow) and Apache Ignite partitioning. Ignite Dataset is a computation graph operation that might be performed on a remote worker. The remote worker can override Ignite Dataset parameters (such as `host`, `port` or `part`) by setting correstondent environment variables for worker process (such as `IGNITE_DATASET_HOST`, `IGNITE_DATASET_PORT` or `IGNITE_DATASET_PART`). Using this overriding approach we are able to assign specific partition to every worker so that one worker handles one partition and, at the same time, transparently work with single dataset. +Ignite Dataset allows using these two aspects of distributed neural network training (using TensorFlow) and Apache Ignite partitioning. Ignite Dataset is a computation graph operation that can be performed on a remote worker. The remote worker can override Ignite Dataset parameters (such as `host`, `port` or `part`) by setting correstondent environment variables for worker process (such as `IGNITE_DATASET_HOST`, `IGNITE_DATASET_PORT` or `IGNITE_DATASET_PART`). Using this overriding approach, we can assign a specific partition to every worker so that one worker handles one partition and, at the same time, transparently work with single dataset. ```python >>> import tensorflow as tf @@ -135,7 +136,7 @@ High-level TensorFlow API for [distributed training](https://www.tensorflow.org/ ### SSL Connection -Your data should not be accessible without any control. Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security) and authentification. Ignite Dataset supports both SSL connection with and without authntication. For more information please see [Apache Ignite SSL/TLS](https://apacheignite.readme.io/docs/ssltls) documentation. +Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security) and authentification. Ignite Dataset supports both SSL connection with and without authntication. For more information, please refer to the [Apache Ignite SSL/TLS](https://apacheignite.readme.io/docs/ssltls) documentation. ```python >>> import tensorflow as tf @@ -147,11 +148,11 @@ Your data should not be accessible without any control. Apache Ignite allows to ### Windows Support -Ignite Dataset is fully compatible with Windows, so you can use it as part of TensorFlow on your Windows workstation as well as on Linux/MacOS systems. +Ignite Dataset is fully compatible with Windows. You can use it as part of TensorFlow on your Windows workstation as well as on Linux/MacOS systems. ## Try it out -The simplest way to try Ignite Dataset out is to run [Docker](https://www.docker.com/) container with Apache Ignite and loaded [MNIST](http://yann.lecun.com/exdb/mnist/) data and then interruct with it using Ignite Dataset. Such container is available on Docker Hub: [dmitrievanthony/ignite-with-mnist](https://hub.docker.com/r/dmitrievanthony/ignite-with-mnist/). You need to start this container on your machine: +The simplest way to try Ignite Dataset is to run a [Docker](https://www.docker.com/) container with Apache Ignite and loaded [MNIST](http://yann.lecun.com/exdb/mnist/) data and after start interruct with it using Ignite Dataset. Such container is available on Docker Hub: [dmitrievanthony/ignite-with-mnist](https://hub.docker.com/r/dmitrievanthony/ignite-with-mnist/). You need to start this container on your machine: ``` docker run -it -p 10800:10800 dmitrievanthony/ignite-with-mnist @@ -163,4 +164,4 @@ After that you will be able to work with it following way: ## Limitations -Presently Ignite Dataset works with assumption that all objects in the cache have the same structure (homogeneous objects) and the cache contains at least one object. Another limitation concerns structured objects, Ignite Dataset does not support UUID, Maps and Object arrays that might be parts of object structures. +Presently, Ignite Dataset works with assumption that all objects in the cache have the same structure (homogeneous objects) and the cache contains at least one object. Another limitation concerns structured objects, Ignite Dataset does not support UUID, Maps and Object arrays that might be parts of an object structure. -- GitLab From 92019765d7b7db99d0235268d00f349b7a53d1a9 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Wed, 5 Sep 2018 14:47:20 +0000 Subject: [PATCH 0048/1085] Fix pylint checks, fix VS compilation issue. --- .../contrib/ignite/kernels/ignite_plain_client_windows.cc | 4 ++-- .../contrib/ignite/python/ops/ignite_dataset_ops.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index 7ba037f2d2..e1e2ee3b20 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -103,7 +103,7 @@ Status PlainClient::ReadData(uint8_t *buf, int32_t length) { int recieved = 0; while (recieved < length) { - int res = recv(sock_, buf, length - recieved, 0); + int res = recv(sock_, (char*)buf, length - recieved, 0); if (res < 0) return errors::Internal("Error occured while reading from socket: ", res); @@ -121,7 +121,7 @@ Status PlainClient::WriteData(uint8_t *buf, int32_t length) { int sent = 0; while (sent < length) { - int res = send(sock_, buf, length - sent, 0); + int res = send(sock_, (char*)buf, length - sent, 0); if (res < 0) return errors::Internal("Error occured while writing into socket: ", res); diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py index 6fa073957a..60003ca3b7 100644 --- a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py +++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py @@ -66,13 +66,13 @@ class Readable(): def __read(self, data_type, length): """Reads, unpacks and returns specified type (little-endian).""" - buffer = self.read_data(length) - return struct.unpack("<" + data_type, buffer)[0] + data_buffer = self.read_data(length) + return struct.unpack("<" + data_type, data_buffer)[0] class DataBuffer(Readable): """DataBuffer class that exposes methods to read data from a byte buffer.""" - def __init__(self, buffer): + def __init__(self, data_buffer): """Constructs a new instance of DataBuffer based on the specified byte buffer. @@ -80,7 +80,7 @@ class DataBuffer(Readable): buffer: Buffer to be read. """ Readable.__init__(self) - self.buffer = buffer + self.buffer = data_buffer self.ptr = 0 def read_data(self, length): -- GitLab From 0b6654bc223f4f3807209043dc34ccb07b55474e Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Tue, 11 Sep 2018 09:50:47 +0000 Subject: [PATCH 0049/1085] Fix code style. --- .../ignite/kernels/ignite_dataset_ops.cc | 2 +- .../kernels/ignite_plain_client_windows.cc | 4 +-- tensorflow/contrib/ignite/ops/dataset_ops.cc | 34 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc index 89eecf9c14..d03404a460 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_dataset.h" #include +#include "ignite_dataset.h" #include "tensorflow/core/framework/dataset.h" namespace tensorflow { diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index e1e2ee3b20..8182fde6d9 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -103,7 +103,7 @@ Status PlainClient::ReadData(uint8_t *buf, int32_t length) { int recieved = 0; while (recieved < length) { - int res = recv(sock_, (char*)buf, length - recieved, 0); + int res = recv(sock_, (char *)buf, length - recieved, 0); if (res < 0) return errors::Internal("Error occured while reading from socket: ", res); @@ -121,7 +121,7 @@ Status PlainClient::WriteData(uint8_t *buf, int32_t length) { int sent = 0; while (sent < length) { - int res = send(sock_, (char*)buf, length - sent, 0); + int res = send(sock_, (char *)buf, length - sent, 0); if (res < 0) return errors::Internal("Error occured while writing into socket: ", res); diff --git a/tensorflow/contrib/ignite/ops/dataset_ops.cc b/tensorflow/contrib/ignite/ops/dataset_ops.cc index 17494d1cfd..fb16b290b1 100644 --- a/tensorflow/contrib/ignite/ops/dataset_ops.cc +++ b/tensorflow/contrib/ignite/ops/dataset_ops.cc @@ -20,23 +20,23 @@ limitations under the License. namespace tensorflow { REGISTER_OP("IgniteDataset") - .Input("cache_name: string") - .Input("host: string") - .Input("port: int32") - .Input("local: bool") - .Input("part: int32") - .Input("page_size: int32") - .Input("username: string") - .Input("password: string") - .Input("certfile: string") - .Input("keyfile: string") - .Input("cert_password: string") - .Input("schema: int32") - .Input("permutation: int32") - .Output("handle: variant") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape) - .Doc(R"doc( + .Input("cache_name: string") + .Input("host: string") + .Input("port: int32") + .Input("local: bool") + .Input("part: int32") + .Input("page_size: int32") + .Input("username: string") + .Input("password: string") + .Input("certfile: string") + .Input("keyfile: string") + .Input("cert_password: string") + .Input("schema: int32") + .Input("permutation: int32") + .Output("handle: variant") + .SetIsStateful() + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( Apache Ignite is a memory-centric distributed database, caching, and processing platform for transactional, analytical, and streaming workloads, delivering in-memory speeds at petabyte scale. This contrib package contains an -- GitLab From 5e9a9547f907599f6954fc5e28b7a78acf3b54eb Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Wed, 12 Sep 2018 11:02:12 +0800 Subject: [PATCH 0050/1085] Revert "Add XLA support for LeakyReluOp." This reverts commit d2ad105d2dff3c79d8f49f5fb8ce74c38f424e74. Since bfloat16 was not supported by LeakyRelu, but it should be supported in XLA Ops. --- tensorflow/compiler/tests/binary_ops_test.py | 8 ---- tensorflow/compiler/tests/unary_ops_test.py | 5 --- tensorflow/compiler/tf2xla/kernels/relu_op.cc | 42 ------------------- 3 files changed, 55 deletions(-) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index c478ff4eea..17280e445b 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -178,14 +178,6 @@ class BinaryOpsTest(xla_test.XLATestCase): [0, 0, 0, 0, 0, 0.1, 0.3, 0.5, 0.7, 0.9, 6.1, 10.0], dtype=dtype), expected=np.array([0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 0, 0], dtype=dtype)) - self._testBinary( - gen_nn_ops.leaky_relu_grad, - np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype), - np.array([-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], - dtype=dtype), - expected=np.array([0.2, 0.4, 0.6, 0.8, 1, 6, 7, 8, 9, 10], - dtype=dtype)) - self._testBinary( gen_nn_ops.softmax_cross_entropy_with_logits, np.array([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=dtype), diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index dd29ef34ce..5b0e57f83f 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -361,11 +361,6 @@ class UnaryOpsTest(xla_test.XLATestCase): np.array([[-0.05, 6.05, 5]], dtype=dtype), expected=np.array([[0, 6, 5]], dtype=dtype)) - self._assertOpOutputMatchesExpected( - nn_ops.leaky_relu, - np.array([[-1.0, 1.0]], dtype=dtype), - expected=np.array([[-0.2, 1.0]], dtype=dtype)) - self._assertOpOutputMatchesExpected( nn_ops.softmax, np.array([1, 2, 3, 4], dtype=dtype), diff --git a/tensorflow/compiler/tf2xla/kernels/relu_op.cc b/tensorflow/compiler/tf2xla/kernels/relu_op.cc index 8d65e0339c..d35777ccb1 100644 --- a/tensorflow/compiler/tf2xla/kernels/relu_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/relu_op.cc @@ -50,23 +50,6 @@ class Relu6Op : public XlaOpKernel { } }; -class LeakyReluOp : public XlaOpKernel { - public: - explicit LeakyReluOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("alpha", &alpha_)); - } - // Compute the max of the input x and alpha*x. - void Compile(XlaOpKernelContext* ctx) override { - xla::XlaBuilder* builder = ctx->builder(); - auto alpha = XlaHelpers::FloatLiteral(builder, input_type(0), - static_cast(alpha_)); - ctx->SetOutput(0, xla::Max(xla::Mul(alpha, ctx->Input(0)), ctx->Input(0))); - } - - private: - float alpha_; -}; - class ReluGradOp : public XlaOpKernel { public: explicit ReluGradOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} @@ -101,35 +84,10 @@ class Relu6GradOp : public XlaOpKernel { } }; -class LeakyReluGradOp : public XlaOpKernel { - public: - explicit LeakyReluGradOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("alpha", &alpha_)); - } - // Return the lhs (incoming gradient) if the rhs (input feature) > 0, - // otherwise return the alpha * lhs. - void Compile(XlaOpKernelContext* ctx) override { - xla::XlaBuilder* b = ctx->builder(); - const TensorShape shape = ctx->InputShape(0); - const auto zero = - xla::Broadcast(XlaHelpers::Zero(b, input_type(0)), shape.dim_sizes()); - const auto pred = xla::Gt(ctx->Input(1), zero); - auto alpha = - XlaHelpers::FloatLiteral(b, input_type(0), static_cast(alpha_)); - ctx->SetOutput( - 0, xla::Select(pred, ctx->Input(0), xla::Mul(alpha, ctx->Input(0)))); - } - - private: - float alpha_; -}; - REGISTER_XLA_OP(Name("Relu"), ReluOp); REGISTER_XLA_OP(Name("Relu6"), Relu6Op); -REGISTER_XLA_OP(Name("LeakyRelu"), LeakyReluOp); REGISTER_XLA_OP(Name("ReluGrad"), ReluGradOp); REGISTER_XLA_OP(Name("Relu6Grad"), Relu6GradOp); -REGISTER_XLA_OP(Name("LeakyReluGrad"), LeakyReluGradOp); } // namespace } // namespace tensorflow -- GitLab From 9ec9c8b24cca5f1e746fef8cd351b3cae6d5a740 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Wed, 12 Sep 2018 20:42:01 +0300 Subject: [PATCH 0051/1085] Fixes after second review. --- tensorflow/contrib/ignite/BUILD | 1 + tensorflow/contrib/ignite/__init__.py | 22 +- .../kernels/ignite_binary_object_parser.cc | 404 ++++++++++-------- .../kernels/ignite_binary_object_parser.h | 36 +- .../contrib/ignite/kernels/ignite_client.h | 55 ++- .../contrib/ignite/kernels/ignite_dataset.cc | 99 ++--- .../contrib/ignite/kernels/ignite_dataset.h | 37 +- .../ignite/kernels/ignite_dataset_iterator.cc | 383 ++++++++--------- .../ignite/kernels/ignite_dataset_iterator.h | 74 ++-- .../ignite/kernels/ignite_dataset_ops.cc | 123 ++++-- .../ignite/kernels/ignite_plain_client.h | 15 +- .../kernels/ignite_plain_client_unix.cc | 14 +- .../kernels/ignite_plain_client_windows.cc | 17 +- .../ignite/kernels/ignite_ssl_wrapper.cc | 34 +- .../ignite/kernels/ignite_ssl_wrapper.h | 26 +- tensorflow/contrib/ignite/ops/dataset_ops.cc | 2 + .../ignite/python/ops/ignite_dataset_ops.py | 176 ++++---- 17 files changed, 848 insertions(+), 670 deletions(-) diff --git a/tensorflow/contrib/ignite/BUILD b/tensorflow/contrib/ignite/BUILD index b7d40a99f7..2f598b4aed 100644 --- a/tensorflow/contrib/ignite/BUILD +++ b/tensorflow/contrib/ignite/BUILD @@ -40,6 +40,7 @@ cc_library( srcs = [ "kernels/ignite_dataset_ops.cc", "kernels/ignite_client.h", + "kernels/ignite_byte_swapper.h", "kernels/ignite_plain_client.h", "kernels/ignite_ssl_wrapper.h", "kernels/ignite_ssl_wrapper.cc", diff --git a/tensorflow/contrib/ignite/__init__.py b/tensorflow/contrib/ignite/__init__.py index b78829d0f4..f42947696f 100644 --- a/tensorflow/contrib/ignite/__init__.py +++ b/tensorflow/contrib/ignite/__init__.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Apache Ignite is a memory-centric distributed database, caching, and - processing platform for transactional, analytical, and streaming workloads, - delivering in-memory speeds at petabyte scale. This contrib package - contains an integration between Apache Ignite and TensorFlow. The - integration is based on tf.data from TensorFlow side and Binary Client - Protocol from Apache Ignite side. It allows to use Apache Ignite as a - datasource for neural network training, inference and all other - computations supported by TensorFlow. Ignite Dataset is based on Apache - Ignite Binary Client Protocol: - https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. +"""IgniteDataset that allows to get data from Apache Ignite. + +Apache Ignite is a memory-centric distributed database, caching, and +processing platform for transactional, analytical, and streaming workloads, +delivering in-memory speeds at petabyte scale. This contrib package +contains an integration between Apache Ignite and TensorFlow. The +integration is based on tf.data from TensorFlow side and Binary Client +Protocol from Apache Ignite side. It allows to use Apache Ignite as a +datasource for neural network training, inference and all other +computations supported by TensorFlow. Ignite Dataset is based on Apache +Ignite Binary Client Protocol: +https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. @@IgniteDataset """ diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc index 9bf4480d2d..2c8a7d44b0 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.cc @@ -13,242 +13,171 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_binary_object_parser.h" +#include "tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" namespace tensorflow { +BinaryObjectParser::BinaryObjectParser() : byte_swapper_(ByteSwapper(false)) {} + Status BinaryObjectParser::Parse(uint8_t** ptr, std::vector* out_tensors, - std::vector* types) { - uint8_t object_type_id = **ptr; - *ptr += 1; + std::vector* types) const { + uint8_t object_type_id = ParseByte(ptr); + + // Skip non-leaf nodes. + if (object_type_id != WRAPPED_OBJ && object_type_id != COMPLEX_OBJ) + types->push_back(object_type_id); switch (object_type_id) { case BYTE: { - Tensor tensor(cpu_allocator(), DT_UINT8, {}); - tensor.scalar()() = *((uint8_t*)*ptr); - *ptr += 1; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_UINT8, TensorShape({})); + out_tensors->back().scalar()() = ParseByte(ptr); break; } case SHORT: { - Tensor tensor(cpu_allocator(), DT_INT16, {}); - tensor.scalar()() = *((int16_t*)*ptr); - *ptr += 2; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_INT16, TensorShape({})); + out_tensors->back().scalar()() = ParseShort(ptr); + break; + } + case USHORT: { + out_tensors->emplace_back(cpu_allocator(), DT_UINT16, TensorShape({})); + out_tensors->back().scalar()() = ParseUnsignedShort(ptr); break; } case INT: { - Tensor tensor(cpu_allocator(), DT_INT32, {}); - tensor.scalar()() = *((int32_t*)*ptr); - *ptr += 4; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_INT32, TensorShape({})); + out_tensors->back().scalar()() = ParseInt(ptr); break; } case LONG: { - Tensor tensor(cpu_allocator(), DT_INT64, {}); - tensor.scalar()() = *((int64_t*)*ptr); - *ptr += 8; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_INT64, TensorShape({})); + out_tensors->back().scalar()() = ParseLong(ptr); break; } case FLOAT: { - Tensor tensor(cpu_allocator(), DT_FLOAT, {}); - tensor.scalar()() = *((float*)*ptr); - *ptr += 4; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_FLOAT, TensorShape({})); + out_tensors->back().scalar()() = ParseFloat(ptr); break; } case DOUBLE: { - Tensor tensor(cpu_allocator(), DT_DOUBLE, {}); - tensor.scalar()() = *((double*)*ptr); - *ptr += 8; - out_tensors->push_back(std::move(tensor)); - break; - } - case UCHAR: { - Tensor tensor(cpu_allocator(), DT_UINT16, {}); - tensor.scalar()() = *((uint16_t*)*ptr); - *ptr += 2; - out_tensors->push_back(std::move(tensor)); + out_tensors->emplace_back(cpu_allocator(), DT_DOUBLE, TensorShape({})); + out_tensors->back().scalar()() = ParseDouble(ptr); break; } case BOOL: { - Tensor tensor(cpu_allocator(), DT_BOOL, {}); - tensor.scalar()() = *((bool*)*ptr); - *ptr += 1; - out_tensors->push_back(std::move(tensor)); - + out_tensors->emplace_back(cpu_allocator(), DT_BOOL, TensorShape({})); + out_tensors->back().scalar()() = ParseBool(ptr); break; } case STRING: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_STRING, {}); - tensor.scalar()() = std::string((char*)*ptr, length); - *ptr += length; - out_tensors->push_back(std::move(tensor)); - + out_tensors->emplace_back(cpu_allocator(), DT_STRING, TensorShape({})); + out_tensors->back().scalar()() = ParseString(ptr); break; } case DATE: { - Tensor tensor(cpu_allocator(), DT_INT64, {}); - tensor.scalar()() = *((int64_t*)*ptr); - *ptr += 8; - out_tensors->push_back(std::move(tensor)); - + out_tensors->emplace_back(cpu_allocator(), DT_INT64, TensorShape({})); + out_tensors->back().scalar()() = ParseLong(ptr); break; } case BYTE_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_UINT8, TensorShape({length})); - - uint8_t* arr = (uint8_t*)*ptr; - *ptr += length; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + uint8_t* arr = ParseByteArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_UINT8, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case SHORT_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_INT16, TensorShape({length})); - - int16_t* arr = (int16_t*)*ptr; - *ptr += length * 2; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + int16_t* arr = ParseShortArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_INT16, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); + break; + } + case USHORT_ARR: { + int32_t length = ParseInt(ptr); + uint16_t* arr = ParseUnsignedShortArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_UINT16, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case INT_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_INT32, TensorShape({length})); - - int32_t* arr = (int32_t*)*ptr; - *ptr += length * 4; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + int32_t* arr = ParseIntArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_INT32, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case LONG_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_INT64, TensorShape({length})); - - int64_t* arr = (int64_t*)*ptr; - *ptr += length * 8; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + int64_t* arr = ParseLongArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_INT64, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case FLOAT_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_FLOAT, TensorShape({length})); - - float* arr = (float*)*ptr; - *ptr += 4 * length; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + float* arr = ParseFloatArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_FLOAT, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case DOUBLE_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_DOUBLE, TensorShape({length})); - - double* arr = (double*)*ptr; - *ptr += 8 * length; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); - break; - } - case UCHAR_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_UINT16, TensorShape({length})); - - uint16_t* arr = (uint16_t*)*ptr; - *ptr += length * 2; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + double* arr = ParseDoubleArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_DOUBLE, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case BOOL_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_BOOL, TensorShape({length})); - - bool* arr = (bool*)*ptr; - *ptr += length; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + bool* arr = ParseBoolArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_BOOL, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case STRING_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_STRING, TensorShape({length})); - - for (int32_t i = 0; i < length; i++) { - int32_t str_length = *((int32_t*)*ptr); - *ptr += 4; - const int8_t* str = (const int8_t*)*ptr; - *ptr += str_length; - tensor.vec()(i) = std::string((char*)str, str_length); - } - - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + out_tensors->emplace_back(cpu_allocator(), DT_STRING, + TensorShape({length})); + for (int32_t i = 0; i < length; i++) + out_tensors->back().vec()(i) = ParseString(ptr); break; } case DATE_ARR: { - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - Tensor tensor(cpu_allocator(), DT_INT64, TensorShape({length})); - int64_t* arr = (int64_t*)*ptr; - *ptr += length * 8; - - std::copy_n(arr, length, tensor.flat().data()); - out_tensors->push_back(std::move(tensor)); + int32_t length = ParseInt(ptr); + int64_t* arr = ParseLongArr(ptr, length); + out_tensors->emplace_back(cpu_allocator(), DT_INT64, + TensorShape({length})); + std::copy_n(arr, length, out_tensors->back().flat().data()); break; } case WRAPPED_OBJ: { - int32_t byte_arr_size = *((int32_t*)*ptr); - *ptr += 4; - + int32_t byte_arr_size = ParseInt(ptr); TF_RETURN_IF_ERROR(Parse(ptr, out_tensors, types)); - - int32_t offset = *((int32_t*)*ptr); - *ptr += 4; + int32_t offset = ParseInt(ptr); break; } case COMPLEX_OBJ: { - uint8_t version = **ptr; - *ptr += 1; - int16_t flags = *((int16_t*)*ptr); // USER_TYPE = 1, HAS_SCHEMA = 2 - *ptr += 2; - int32_t type_id = *((int32_t*)*ptr); - *ptr += 4; - int32_t hash_code = *((int32_t*)*ptr); - *ptr += 4; - int32_t length = *((int32_t*)*ptr); - *ptr += 4; - int32_t schema_id = *((int32_t*)*ptr); - *ptr += 4; - int32_t schema_offset = *((int32_t*)*ptr); - *ptr += 4; - + uint8_t version = ParseByte(ptr); + int16_t flags = ParseShort(ptr); + int32_t type_id = ParseInt(ptr); + int32_t hash_code = ParseInt(ptr); + int32_t length = ParseInt(ptr); + int32_t schema_id = ParseInt(ptr); + int32_t schema_offset = ParseInt(ptr); + + // 24 is size of header just read. uint8_t* end = *ptr + schema_offset - 24; int32_t i = 0; while (*ptr < end) { @@ -261,12 +190,145 @@ Status BinaryObjectParser::Parse(uint8_t** ptr, break; } default: { - return errors::Internal("Unknowd binary type (type id ", - (int)object_type_id, ")"); + return errors::Unknown("Unknowd binary type (type id ", + (int)object_type_id, ")"); } } return Status::OK(); } +uint8_t BinaryObjectParser::ParseByte(uint8_t** ptr) const { + uint8_t res = **ptr; + *ptr += 1; + + return res; +} + +int16_t BinaryObjectParser::ParseShort(uint8_t** ptr) const { + int16_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt16(res); + *ptr += 2; + + return *res; +} + +uint16_t BinaryObjectParser::ParseUnsignedShort(uint8_t** ptr) const { + uint16_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredUnsignedInt16(res); + *ptr += 2; + + return *res; +} + +int32_t BinaryObjectParser::ParseInt(uint8_t** ptr) const { + int32_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt32(res); + *ptr += 4; + + return *res; +} + +int64_t BinaryObjectParser::ParseLong(uint8_t** ptr) const { + int64_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt64(res); + *ptr += 8; + + return *res; +} + +float BinaryObjectParser::ParseFloat(uint8_t** ptr) const { + float* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredFloat(res); + *ptr += 4; + + return *res; +} + +double BinaryObjectParser::ParseDouble(uint8_t** ptr) const { + double* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredDouble(res); + *ptr += 8; + + return *res; +} + +bool BinaryObjectParser::ParseBool(uint8_t** ptr) const { + bool res = **reinterpret_cast(ptr); + *ptr += 1; + + return res; +} + +string BinaryObjectParser::ParseString(uint8_t** ptr) const { + int32_t length = ParseInt(ptr); + string res(*reinterpret_cast(ptr), length); + *ptr += length; + + return res; +} + +uint8_t* BinaryObjectParser::ParseByteArr(uint8_t** ptr, int length) const { + uint8_t* res = *reinterpret_cast(ptr); + *ptr += length; + + return res; +} + +int16_t* BinaryObjectParser::ParseShortArr(uint8_t** ptr, int length) const { + int16_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt16Arr(res, length); + *ptr += length * 2; + + return res; +} + +uint16_t* BinaryObjectParser::ParseUnsignedShortArr(uint8_t** ptr, + int length) const { + uint16_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredUnsignedInt16Arr(res, length); + *ptr += length * 2; + + return res; +} + +int32_t* BinaryObjectParser::ParseIntArr(uint8_t** ptr, int length) const { + int32_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt32Arr(res, length); + *ptr += length * 4; + + return res; +} + +int64_t* BinaryObjectParser::ParseLongArr(uint8_t** ptr, int length) const { + int64_t* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredInt64Arr(res, length); + *ptr += length * 8; + + return res; +} + +float* BinaryObjectParser::ParseFloatArr(uint8_t** ptr, int length) const { + float* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredFloatArr(res, length); + *ptr += length * 4; + + return res; +} + +double* BinaryObjectParser::ParseDoubleArr(uint8_t** ptr, int length) const { + double* res = *reinterpret_cast(ptr); + byte_swapper_.SwapIfRequiredDoubleArr(res, length); + *ptr += length * 8; + + return res; +} + +bool* BinaryObjectParser::ParseBoolArr(uint8_t** ptr, int length) const { + bool* res = *reinterpret_cast(ptr); + *ptr += length; + + return res; +} + } // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h index 9accbd796f..eb1f856643 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h +++ b/tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h @@ -13,16 +13,42 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BINARY_OBJECT_PARSER_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BINARY_OBJECT_PARSER_H_ + #include -#include "tensorflow/core/framework/dataset.h" +#include "tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h" +#include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/status.h" namespace tensorflow { class BinaryObjectParser { public: + BinaryObjectParser(); Status Parse(uint8_t** ptr, std::vector* out_tensors, - std::vector* types); + std::vector* types) const; + + private: + uint8_t ParseByte(uint8_t** ptr) const; + int16_t ParseShort(uint8_t** ptr) const; + uint16_t ParseUnsignedShort(uint8_t** ptr) const; + int32_t ParseInt(uint8_t** ptr) const; + int64_t ParseLong(uint8_t** ptr) const; + float ParseFloat(uint8_t** ptr) const; + double ParseDouble(uint8_t** ptr) const; + bool ParseBool(uint8_t** ptr) const; + string ParseString(uint8_t** ptr) const; + uint8_t* ParseByteArr(uint8_t** ptr, int length) const; + int16_t* ParseShortArr(uint8_t** ptr, int length) const; + uint16_t* ParseUnsignedShortArr(uint8_t** ptr, int length) const; + int32_t* ParseIntArr(uint8_t** ptr, int length) const; + int64_t* ParseLongArr(uint8_t** ptr, int length) const; + float* ParseFloatArr(uint8_t** ptr, int length) const; + double* ParseDoubleArr(uint8_t** ptr, int length) const; + bool* ParseBoolArr(uint8_t** ptr, int length) const; + + const ByteSwapper byte_swapper_; }; enum ObjectType { @@ -32,7 +58,7 @@ enum ObjectType { LONG = 4, FLOAT = 5, DOUBLE = 6, - UCHAR = 7, + USHORT = 7, BOOL = 8, STRING = 9, DATE = 11, @@ -42,7 +68,7 @@ enum ObjectType { LONG_ARR = 15, FLOAT_ARR = 16, DOUBLE_ARR = 17, - UCHAR_ARR = 18, + USHORT_ARR = 18, BOOL_ARR = 19, STRING_ARR = 20, DATE_ARR = 22, @@ -51,3 +77,5 @@ enum ObjectType { }; } // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BINARY_OBJECT_PARSER_H_ diff --git a/tensorflow/contrib/ignite/kernels/ignite_client.h b/tensorflow/contrib/ignite/kernels/ignite_client.h index 944b3fe184..508b6e4a60 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_client.h +++ b/tensorflow/contrib/ignite/kernels/ignite_client.h @@ -16,40 +16,69 @@ limitations under the License. #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_CLIENT_H_ #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_CLIENT_H_ +#include "tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h" +#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" namespace tensorflow { class Client { public: + Client(bool big_endian) : byte_swapper_(ByteSwapper(big_endian)){}; virtual Status Connect() = 0; virtual Status Disconnect() = 0; virtual bool IsConnected() = 0; virtual int GetSocketDescriptor() = 0; - virtual Status ReadData(uint8_t* buf, int32_t length) = 0; - virtual Status WriteData(uint8_t* buf, int32_t length) = 0; + virtual Status ReadData(uint8_t *buf, const int32_t length) = 0; + virtual Status WriteData(const uint8_t *buf, const int32_t length) = 0; - inline Status ReadByte(uint8_t* data) { return ReadData(data, 1); } + inline Status ReadByte(uint8_t *data) { return ReadData(data, 1); } - inline Status ReadShort(int16_t* data) { return ReadData((uint8_t*)data, 2); } + inline Status ReadShort(int16_t *data) { + TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 2)); + byte_swapper_.SwapIfRequiredInt16(data); - inline Status ReadInt(int32_t* data) { return ReadData((uint8_t*)data, 4); } + return Status::OK(); + } + + inline Status ReadInt(int32_t *data) { + TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 4)); + byte_swapper_.SwapIfRequiredInt32(data); + + return Status::OK(); + } - inline Status ReadLong(int64_t* data) { return ReadData((uint8_t*)data, 8); } + inline Status ReadLong(int64_t *data) { + TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 8)); + byte_swapper_.SwapIfRequiredInt64(data); - inline Status WriteByte(uint8_t data) { return WriteData(&data, 1); } + return Status::OK(); + } + + inline Status WriteByte(const uint8_t data) { return WriteData(&data, 1); } - inline Status WriteShort(int16_t data) { - return WriteData((uint8_t*)&data, 2); + inline Status WriteShort(const int16_t data) { + int16_t tmp = data; + byte_swapper_.SwapIfRequiredInt16(&tmp); + return WriteData((uint8_t *)&tmp, 2); } - inline Status WriteInt(int32_t data) { return WriteData((uint8_t*)&data, 4); } + inline Status WriteInt(const int32_t data) { + int32_t tmp = data; + byte_swapper_.SwapIfRequiredInt32(&tmp); + return WriteData((uint8_t *)&tmp, 4); + } - inline Status WriteLong(int64_t data) { - return WriteData((uint8_t*)&data, 8); + inline Status WriteLong(const int64_t data) { + int64_t tmp = data; + byte_swapper_.SwapIfRequiredInt64(&tmp); + return WriteData((uint8_t *)&tmp, 8); } + + private: + const ByteSwapper byte_swapper_; }; } // namespace tensorflow -#endif +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_CLIENT_H_ diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc index f25f8a5b18..c4a7d3c513 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.cc @@ -13,40 +13,41 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_dataset_iterator.h" +#include "tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h" #include "tensorflow/core/platform/logging.h" namespace tensorflow { -IgniteDataset::IgniteDataset(OpKernelContext* ctx, std::string cache_name, - std::string host, int32 port, bool local, - int32 part, int32 page_size, std::string username, - std::string password, std::string certfile, - std::string keyfile, std::string cert_password, - std::vector schema, - std::vector permutation) +IgniteDataset::IgniteDataset(OpKernelContext* ctx, string cache_name, + string host, int32 port, bool local, int32 part, + int32 page_size, string username, string password, + string certfile, string keyfile, + string cert_password, std::vector schema, + std::vector permutation, + DataTypeVector dtypes, + std::vector shapes) : DatasetBase(DatasetContext(ctx)), - cache_name_(cache_name), - host_(host), + cache_name_(std::move(cache_name)), + host_(std::move(host)), port_(port), local_(local), part_(part), page_size_(page_size), - username_(username), - password_(password), - certfile_(certfile), - keyfile_(keyfile), - cert_password_(cert_password), - schema_(schema), - permutation_(permutation) { - SchemaToTypes(); - SchemaToShapes(); - - LOG(INFO) << "Ignite Dataset created [cache_name='" << cache_name - << "', host='" << host << "', port=" << port << ", local=" << local - << ", part=" << part << ", page_size=" << page_size - << ", username='" << username << "', certfile='" << certfile - << "', keyfile='" << keyfile + "']"; + username_(std::move(username)), + password_(std::move(password)), + certfile_(std::move(certfile)), + keyfile_(std::move(keyfile)), + cert_password_(std::move(cert_password)), + schema_(std::move(schema)), + permutation_(std::move(permutation)), + dtypes_(dtypes), + shapes_(shapes) { + LOG(INFO) << "Ignite Dataset created [cache_name='" << cache_name_ + << "', host='" << host_ << "', port=" << port_ + << ", local=" << local_ << ", part=" << part_ + << ", page_size=" << page_size_ << ", username='" << username_ + << "', certfile='" << certfile_ << "', keyfile='" + << keyfile_ + "']"; } IgniteDataset::~IgniteDataset() { LOG(INFO) << "Ignite Dataset destroyed"; } @@ -54,10 +55,12 @@ IgniteDataset::~IgniteDataset() { LOG(INFO) << "Ignite Dataset destroyed"; } std::unique_ptr IgniteDataset::MakeIteratorInternal( const string& prefix) const { return std::unique_ptr(new IgniteDatasetIterator( - {this, strings::StrCat(prefix, "::Ignite")}, this->host_, this->port_, - this->cache_name_, this->local_, this->part_, this->page_size_, - this->username_, this->password_, this->certfile_, this->keyfile_, - this->cert_password_, this->schema_, this->permutation_)); + {this, strings::StrCat(prefix, "::Ignite")}, std::move(this->host_), + this->port_, std::move(this->cache_name_), this->local_, this->part_, + this->page_size_, std::move(this->username_), std::move(this->password_), + std::move(this->certfile_), std::move(this->keyfile_), + std::move(this->cert_password_), std::move(this->schema_), + std::move(this->permutation_))); } const DataTypeVector& IgniteDataset::output_dtypes() const { return dtypes_; } @@ -75,42 +78,4 @@ Status IgniteDataset::AsGraphDefInternal(SerializationContext* ctx, "IgniteDataset does not support 'AsGraphDefInternal'"); } -void IgniteDataset::SchemaToTypes() { - for (auto e : schema_) { - if (e == BYTE || e == BYTE_ARR) { - dtypes_.push_back(DT_UINT8); - } else if (e == SHORT || e == SHORT_ARR) { - dtypes_.push_back(DT_INT16); - } else if (e == INT || e == INT_ARR) { - dtypes_.push_back(DT_INT32); - } else if (e == LONG || e == LONG_ARR) { - dtypes_.push_back(DT_INT64); - } else if (e == FLOAT || e == FLOAT_ARR) { - dtypes_.push_back(DT_FLOAT); - } else if (e == DOUBLE || e == DOUBLE_ARR) { - dtypes_.push_back(DT_DOUBLE); - } else if (e == UCHAR || e == UCHAR_ARR) { - dtypes_.push_back(DT_UINT8); - } else if (e == BOOL || e == BOOL_ARR) { - dtypes_.push_back(DT_BOOL); - } else if (e == STRING || e == STRING_ARR) { - dtypes_.push_back(DT_STRING); - } else { - LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; - } - } -} - -void IgniteDataset::SchemaToShapes() { - for (auto e : schema_) { - if (e >= 1 && e < 10) { - shapes_.push_back(PartialTensorShape({})); - } else if (e >= 12 && e < 21) { - shapes_.push_back(PartialTensorShape({-1})); - } else { - LOG(ERROR) << "Unexpected type in schema [type_id=" << e << "]"; - } - } -} - } // namespace tensorflow diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset.h b/tensorflow/contrib/ignite/kernels/ignite_dataset.h index d3fec5910b..66bfdf2e2a 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset.h +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset.h @@ -13,18 +13,21 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_H_ + #include "tensorflow/core/framework/dataset.h" namespace tensorflow { class IgniteDataset : public DatasetBase { public: - IgniteDataset(OpKernelContext* ctx, std::string cache_name, std::string host, + IgniteDataset(OpKernelContext* ctx, string cache_name, string host, int32 port, bool local, int32 part, int32 page_size, - std::string username, std::string password, - std::string certfile, std::string keyfile, - std::string cert_password, std::vector schema, - std::vector permutation); + string username, string password, string certfile, + string keyfile, string cert_password, std::vector schema, + std::vector permutation, DataTypeVector dtypes, + std::vector shapes); ~IgniteDataset(); std::unique_ptr MakeIteratorInternal( const string& prefix) const override; @@ -38,25 +41,23 @@ class IgniteDataset : public DatasetBase { Node** output) const override; private: - const std::string cache_name_; - const std::string host_; + const string cache_name_; + const string host_; const int32 port_; const bool local_; const int32 part_; const int32 page_size_; - const std::string username_; - const std::string password_; - const std::string certfile_; - const std::string keyfile_; - const std::string cert_password_; + const string username_; + const string password_; + const string certfile_; + const string keyfile_; + const string cert_password_; const std::vector schema_; const std::vector permutation_; - - DataTypeVector dtypes_; - std::vector shapes_; - - void SchemaToTypes(); - void SchemaToShapes(); + const DataTypeVector dtypes_; + const std::vector shapes_; }; } // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_H_ diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc index 1774585ecd..f68ded5a3a 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.cc @@ -13,10 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_dataset_iterator.h" - -#include "ignite_plain_client.h" -#include "ignite_ssl_wrapper.h" +#include "tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h" +#include "tensorflow/contrib/ignite/kernels/ignite_plain_client.h" +#include "tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/platform/logging.h" #include @@ -25,30 +25,31 @@ limitations under the License. namespace tensorflow { IgniteDatasetIterator::IgniteDatasetIterator( - const Params& params, std::string host, int32 port, std::string cache_name, - bool local, int32 part, int32 page_size, std::string username, - std::string password, std::string certfile, std::string keyfile, - std::string cert_password, std::vector schema, - std::vector permutation) + const Params& params, string host, int32 port, string cache_name, + bool local, int32 part, int32 page_size, string username, string password, + string certfile, string keyfile, string cert_password, + std::vector schema, std::vector permutation) : DatasetIterator(params), - cache_name_(cache_name), + cache_name_(std::move(cache_name)), local_(local), part_(part), page_size_(page_size), - username_(username), - password_(password), - schema_(schema), - permutation_(permutation), + username_(std::move(username)), + password_(std::move(password)), + schema_(std::move(schema)), + permutation_(std::move(permutation)), remainder_(-1), cursor_id_(-1), - last_page_(false) { - Client* p_client = new PlainClient(host, port); + last_page_(false), + valid_state_(true) { + Client* p_client = new PlainClient(std::move(host), port, false); if (certfile.empty()) client_ = std::unique_ptr(p_client); else - client_ = std::unique_ptr(new SslWrapper( - std::unique_ptr(p_client), certfile, keyfile, cert_password)); + client_ = std::unique_ptr( + new SslWrapper(std::unique_ptr(p_client), std::move(certfile), + std::move(keyfile), std::move(cert_password), false)); LOG(INFO) << "Ignite Dataset Iterator created"; } @@ -60,12 +61,80 @@ IgniteDatasetIterator::~IgniteDatasetIterator() { LOG(INFO) << "Ignite Dataset Iterator destroyed"; } +Status IgniteDatasetIterator::GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) { + mutex_lock l(mutex_); + + if (valid_state_) { + Status status = + GetNextInternalWithValidState(ctx, out_tensors, end_of_sequence); + + if (!status.ok()) valid_state_ = false; + + return status; + } + + return errors::Unknown("Iterator is invalid"); +} + +Status IgniteDatasetIterator::SaveInternal(IteratorStateWriter* writer) { + return errors::Unimplemented( + "Iterator for IgniteDataset does not support 'SaveInternal'"); +} + +Status IgniteDatasetIterator::RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) { + return errors::Unimplemented( + "Iterator for IgniteDataset does not support 'RestoreInternal')"); +} + +Status IgniteDatasetIterator::GetNextInternalWithValidState( + IteratorContext* ctx, std::vector* out_tensors, + bool* end_of_sequence) { + if (remainder_ == 0 && last_page_) { + cursor_id_ = -1; + *end_of_sequence = true; + + return Status::OK(); + } else { + TF_RETURN_IF_ERROR(EstablishConnection()); + + if (remainder_ == -1) { + TF_RETURN_IF_ERROR(ScanQuery()); + } else if (remainder_ == 0) { + TF_RETURN_IF_ERROR(LoadNextPage()); + } + + uint8_t* initial_ptr = ptr_; + std::vector tensors; + std::vector types; + + TF_RETURN_IF_ERROR(parser_.Parse(&ptr_, &tensors, &types)); // Parse key + TF_RETURN_IF_ERROR(parser_.Parse(&ptr_, &tensors, &types)); // Parse val + + remainder_ -= (ptr_ - initial_ptr); + + TF_RETURN_IF_ERROR(CheckTypes(types)); + + for (size_t i = 0; i < tensors.size(); i++) + out_tensors->push_back(tensors[permutation_[i]]); + + *end_of_sequence = false; + + return Status::OK(); + } + + *end_of_sequence = true; + + return Status::OK(); +} + Status IgniteDatasetIterator::EstablishConnection() { if (!client_->IsConnected()) { - Status status = client_->Connect(); - if (!status.ok()) return status; + TF_RETURN_IF_ERROR(client_->Connect()); - status = Handshake(); + Status status = Handshake(); if (!status.ok()) { Status disconnect_status = client_->Disconnect(); if (!disconnect_status.ok()) LOG(ERROR) << disconnect_status.ToString(); @@ -79,19 +148,17 @@ Status IgniteDatasetIterator::EstablishConnection() { Status IgniteDatasetIterator::CloseConnection() { if (cursor_id_ != -1 && !last_page_) { - Status conn_status = EstablishConnection(); - if (!conn_status.ok()) return conn_status; + TF_RETURN_IF_ERROR(EstablishConnection()); - TF_RETURN_IF_ERROR(client_->WriteInt(18)); // Message length - TF_RETURN_IF_ERROR( - client_->WriteShort(close_connection_opcode)); // Operation code + TF_RETURN_IF_ERROR(client_->WriteInt(kCloseConnectionReqLength)); + TF_RETURN_IF_ERROR(client_->WriteShort(kCloseConnectionOpcode)); TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID TF_RETURN_IF_ERROR(client_->WriteLong(cursor_id_)); // Resource ID int32_t res_len; TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); - if (res_len < 12) - return errors::Internal("Close Resource Response is corrupted"); + if (res_len < kMinResLength) + return errors::Unknown("Close Resource Response is corrupted"); int64_t req_id; TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); @@ -100,22 +167,21 @@ Status IgniteDatasetIterator::CloseConnection() { if (status != 0) { uint8_t err_msg_header; TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); - if (err_msg_header == string_val) { + if (err_msg_header == kStringVal) { int32_t err_msg_length; TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); + uint8_t* err_msg_c = new uint8_t[err_msg_length]; + auto clean = gtl::MakeCleanup([err_msg_c] { delete[] err_msg_c; }); TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); - std::string err_msg((char*)err_msg_c, err_msg_length); - delete[] err_msg_c; + string err_msg(reinterpret_cast(err_msg_c), err_msg_length); - return errors::Internal("Close Resource Error [status=", status, - ", message=", err_msg, "]"); + return errors::Unknown("Close Resource Error [status=", status, + ", message=", err_msg, "]"); } - return errors::Internal("Close Resource Error [status=", status, "]"); + return errors::Unknown("Close Resource Error [status=", status, "]"); } - LOG(INFO) << "Query Cursor " << cursor_id_ << " is closed"; - cursor_id_ = -1; return client_->Disconnect(); @@ -126,94 +192,43 @@ Status IgniteDatasetIterator::CloseConnection() { return client_->IsConnected() ? client_->Disconnect() : Status::OK(); } -Status IgniteDatasetIterator::GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) { - if (remainder_ == 0 && last_page_) { - LOG(INFO) << "Query Cursor " << cursor_id_ << " is closed"; - - cursor_id_ = -1; - *end_of_sequence = true; - return Status::OK(); - } else { - Status status = EstablishConnection(); - if (!status.ok()) return status; - - if (remainder_ == -1 || remainder_ == 0) { - Status status = remainder_ == -1 ? ScanQuery() : LoadNextPage(); - if (!status.ok()) return status; - } - - uint8_t* initial_ptr = ptr_; - std::vector types; - std::vector tensors; - - status = parser_.Parse(&ptr_, &tensors, &types); // Parse key - if (!status.ok()) return status; - - status = parser_.Parse(&ptr_, &tensors, &types); // Parse val - if (!status.ok()) return status; - - remainder_ -= (ptr_ - initial_ptr); - - out_tensors->resize(tensors.size()); - for (int32_t i = 0; i < tensors.size(); i++) - (*out_tensors)[permutation_[i]] = std::move(tensors[i]); - - *end_of_sequence = false; - return Status::OK(); - } - - *end_of_sequence = true; - return Status::OK(); -} - -Status IgniteDatasetIterator::SaveInternal(IteratorStateWriter* writer) { - return errors::Unimplemented( - "Iterator for IgniteDataset does not support 'SaveInternal'"); -} - -Status IgniteDatasetIterator::RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) { - return errors::Unimplemented( - "Iterator for IgniteDataset does not support 'RestoreInternal')"); -} - Status IgniteDatasetIterator::Handshake() { - int32_t msg_len = 8; + int32_t msg_len = kHandshakeReqDefaultLength; if (username_.empty()) msg_len += 1; else - msg_len += 5 + username_.length(); + msg_len += 5 + username_.length(); // 1 byte header, 4 bytes length. if (password_.empty()) msg_len += 1; else - msg_len += 5 + password_.length(); + msg_len += 5 + password_.length(); // 1 byte header, 4 bytes length. TF_RETURN_IF_ERROR(client_->WriteInt(msg_len)); TF_RETURN_IF_ERROR(client_->WriteByte(1)); - TF_RETURN_IF_ERROR(client_->WriteShort(protocol_major_version)); - TF_RETURN_IF_ERROR(client_->WriteShort(protocol_minor_version)); - TF_RETURN_IF_ERROR(client_->WriteShort(protocol_patch_version)); + TF_RETURN_IF_ERROR(client_->WriteShort(kProtocolMajorVersion)); + TF_RETURN_IF_ERROR(client_->WriteShort(kProtocolMinorVersion)); + TF_RETURN_IF_ERROR(client_->WriteShort(kProtocolPatchVersion)); TF_RETURN_IF_ERROR(client_->WriteByte(2)); if (username_.empty()) { - TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); + TF_RETURN_IF_ERROR(client_->WriteByte(kNullVal)); } else { - TF_RETURN_IF_ERROR(client_->WriteByte(string_val)); + TF_RETURN_IF_ERROR(client_->WriteByte(kStringVal)); TF_RETURN_IF_ERROR(client_->WriteInt(username_.length())); TF_RETURN_IF_ERROR( - client_->WriteData((uint8_t*)username_.c_str(), username_.length())); + client_->WriteData(reinterpret_cast(username_.c_str()), + username_.length())); } if (password_.empty()) { - TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); + TF_RETURN_IF_ERROR(client_->WriteByte(kNullVal)); } else { - TF_RETURN_IF_ERROR(client_->WriteByte(string_val)); + TF_RETURN_IF_ERROR(client_->WriteByte(kStringVal)); TF_RETURN_IF_ERROR(client_->WriteInt(password_.length())); TF_RETURN_IF_ERROR( - client_->WriteData((uint8_t*)password_.c_str(), password_.length())); + client_->WriteData(reinterpret_cast(password_.c_str()), + password_.length())); } int32_t handshake_res_len; @@ -221,9 +236,6 @@ Status IgniteDatasetIterator::Handshake() { uint8_t handshake_res; TF_RETURN_IF_ERROR(client_->ReadByte(&handshake_res)); - LOG(INFO) << "Handshake length " << handshake_res_len << ", res " - << (int16_t)handshake_res; - if (handshake_res != 1) { int16_t serv_ver_major; TF_RETURN_IF_ERROR(client_->ReadShort(&serv_ver_major)); @@ -234,26 +246,26 @@ Status IgniteDatasetIterator::Handshake() { uint8_t header; TF_RETURN_IF_ERROR(client_->ReadByte(&header)); - if (header == string_val) { + if (header == kStringVal) { int32_t length; TF_RETURN_IF_ERROR(client_->ReadInt(&length)); + uint8_t* err_msg_c = new uint8_t[length]; + auto clean = gtl::MakeCleanup([err_msg_c] { delete[] err_msg_c; }); TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, length)); - std::string err_msg((char*)err_msg_c, length); - delete[] err_msg_c; - - return errors::Internal("Handshake Error [result=", handshake_res, - ", version=", serv_ver_major, ".", serv_ver_minor, - ".", serv_ver_patch, ", message='", err_msg, - "']"); - } else if (header == null_val) { - return errors::Internal("Handshake Error [result=", handshake_res, - ", version=", serv_ver_major, ".", serv_ver_minor, - ".", serv_ver_patch, "]"); + string err_msg(reinterpret_cast(err_msg_c), length); + + return errors::Unknown("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, ", message='", err_msg, "']"); + } else if (header == kNullVal) { + return errors::Unknown("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, "]"); } else { - return errors::Internal("Handshake Error [result=", handshake_res, - ", version=", serv_ver_major, ".", serv_ver_minor, - ".", serv_ver_patch, "]"); + return errors::Unknown("Handshake Error [result=", handshake_res, + ", version=", serv_ver_major, ".", serv_ver_minor, + ".", serv_ver_patch, "]"); } } @@ -261,31 +273,26 @@ Status IgniteDatasetIterator::Handshake() { } Status IgniteDatasetIterator::ScanQuery() { - TF_RETURN_IF_ERROR(client_->WriteInt(25)); // Message length - TF_RETURN_IF_ERROR(client_->WriteShort(scan_query_opcode)); // Operation code - TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID + TF_RETURN_IF_ERROR(client_->WriteInt(kScanQueryReqLength)); + TF_RETURN_IF_ERROR(client_->WriteShort(kScanQueryOpcode)); + TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID TF_RETURN_IF_ERROR( client_->WriteInt(JavaHashCode(cache_name_))); // Cache name TF_RETURN_IF_ERROR(client_->WriteByte(0)); // Flags - TF_RETURN_IF_ERROR(client_->WriteByte(null_val)); // Filter object + TF_RETURN_IF_ERROR(client_->WriteByte(kNullVal)); // Filter object TF_RETURN_IF_ERROR(client_->WriteInt(page_size_)); // Cursor page size TF_RETURN_IF_ERROR(client_->WriteInt(part_)); // part_ition to query TF_RETURN_IF_ERROR(client_->WriteByte(local_)); // local_ flag - int64_t wait_start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - + uint64 wait_start = Env::Default()->NowMicros(); int32_t res_len; TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); + int64_t wait_stop = Env::Default()->NowMicros(); - int64_t wait_stop = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); + LOG(INFO) << "Scan Query waited " << (wait_stop - wait_start) / 1000 << " ms"; - LOG(INFO) << "Scan Query waited " << (wait_stop - wait_start) << " ms"; - - if (res_len < 12) return errors::Internal("Scan Query Response is corrupted"); + if (res_len < kMinResLength) + return errors::Unknown("Scan Query Response is corrupted"); int64_t req_id; TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); @@ -297,78 +304,47 @@ Status IgniteDatasetIterator::ScanQuery() { uint8_t err_msg_header; TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); - if (err_msg_header == string_val) { + if (err_msg_header == kStringVal) { int32_t err_msg_length; TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); uint8_t* err_msg_c = new uint8_t[err_msg_length]; + auto clean = gtl::MakeCleanup([err_msg_c] { delete[] err_msg_c; }); TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); - std::string err_msg((char*)err_msg_c, err_msg_length); - delete[] err_msg_c; + string err_msg(reinterpret_cast(err_msg_c), err_msg_length); - return errors::Internal("Scan Query Error [status=", status, ", message=", - err_msg, "]"); + return errors::Unknown("Scan Query Error [status=", status, ", message=", + err_msg, "]"); } - return errors::Internal("Scan Query Error [status=", status, "]"); + return errors::Unknown("Scan Query Error [status=", status, "]"); } TF_RETURN_IF_ERROR(client_->ReadLong(&cursor_id_)); - LOG(INFO) << "Query Cursor " << cursor_id_ << " is opened"; - int32_t row_cnt; TF_RETURN_IF_ERROR(client_->ReadInt(&row_cnt)); - remainder_ = res_len - 25; - page_ = std::unique_ptr(new uint8_t[remainder_]); - ptr_ = page_.get(); - - int64_t start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - - TF_RETURN_IF_ERROR(client_->ReadData(ptr_, remainder_)); - - int64_t stop = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - ; - - double size_in_mb = 1.0 * remainder_ / 1024 / 1024; - double time_in_s = 1.0 * (stop - start) / 1000; - LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 - << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; - - uint8_t last_page_b; - TF_RETURN_IF_ERROR(client_->ReadByte(&last_page_b)); - - last_page_ = !last_page_b; + int32_t page_size = res_len - kScanQueryResHeaderLength; - return Status::OK(); + return ReceivePage(page_size); } Status IgniteDatasetIterator::LoadNextPage() { - TF_RETURN_IF_ERROR(client_->WriteInt(18)); // Message length - TF_RETURN_IF_ERROR( - client_->WriteShort(load_next_page_opcode)); // Operation code + TF_RETURN_IF_ERROR(client_->WriteInt(kLoadNextPageReqLength)); + TF_RETURN_IF_ERROR(client_->WriteShort(kLoadNextPageOpcode)); TF_RETURN_IF_ERROR(client_->WriteLong(0)); // Request ID TF_RETURN_IF_ERROR(client_->WriteLong(cursor_id_)); // Cursor ID - int64_t wait_start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - + uint64 wait_start = Env::Default()->NowMicros(); int32_t res_len; TF_RETURN_IF_ERROR(client_->ReadInt(&res_len)); + uint64 wait_stop = Env::Default()->NowMicros(); - int64_t wait_stop = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); + LOG(INFO) << "Load Next Page waited " << (wait_stop - wait_start) / 1000 + << " ms"; - LOG(INFO) << "Load Next Page waited " << (wait_stop - wait_start) << " ms"; - - if (res_len < 12) - return errors::Internal("Load Next Page Response is corrupted"); + if (res_len < kMinResLength) + return errors::Unknown("Load Next Page Response is corrupted"); int64_t req_id; TF_RETURN_IF_ERROR(client_->ReadLong(&req_id)); @@ -380,41 +356,40 @@ Status IgniteDatasetIterator::LoadNextPage() { uint8_t err_msg_header; TF_RETURN_IF_ERROR(client_->ReadByte(&err_msg_header)); - if (err_msg_header == string_val) { + if (err_msg_header == kStringVal) { int32_t err_msg_length; TF_RETURN_IF_ERROR(client_->ReadInt(&err_msg_length)); uint8_t* err_msg_c = new uint8_t[err_msg_length]; + auto clean = gtl::MakeCleanup([err_msg_c] { delete[] err_msg_c; }); TF_RETURN_IF_ERROR(client_->ReadData(err_msg_c, err_msg_length)); - std::string err_msg((char*)err_msg_c, err_msg_length); - delete[] err_msg_c; + string err_msg(reinterpret_cast(err_msg_c), err_msg_length); - return errors::Internal("Load Next Page Error [status=", status, - ", message=", err_msg, "]"); + return errors::Unknown("Load Next Page Error [status=", status, + ", message=", err_msg, "]"); } - return errors::Internal("Load Next Page Error [status=", status, "]"); + return errors::Unknown("Load Next Page Error [status=", status, "]"); } int32_t row_cnt; TF_RETURN_IF_ERROR(client_->ReadInt(&row_cnt)); - remainder_ = res_len - 17; + int32_t page_size = res_len - kLoadNextPageResHeaderLength; + + return ReceivePage(page_size); +} + +Status IgniteDatasetIterator::ReceivePage(int32_t page_size) { + remainder_ = page_size; page_ = std::unique_ptr(new uint8_t[remainder_]); ptr_ = page_.get(); - int64_t start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - + uint64 start = Env::Default()->NowMicros(); TF_RETURN_IF_ERROR(client_->ReadData(ptr_, remainder_)); - - int64_t stop = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - ; + uint64 stop = Env::Default()->NowMicros(); double size_in_mb = 1.0 * remainder_ / 1024 / 1024; - double time_in_s = 1.0 * (stop - start) / 1000; + double time_in_s = 1.0 * (stop - start) / 1000 / 1000; LOG(INFO) << "Page size " << size_in_mb << " Mb, time " << time_in_s * 1000 << " ms download speed " << size_in_mb / time_in_s << " Mb/sec"; @@ -426,7 +401,19 @@ Status IgniteDatasetIterator::LoadNextPage() { return Status::OK(); } -int32_t IgniteDatasetIterator::JavaHashCode(std::string str) const { +Status IgniteDatasetIterator::CheckTypes(const std::vector& types) { + if (schema_.size() != types.size()) + return errors::Unknown("Object has unexpected schema"); + + for (size_t i = 0; i < schema_.size(); i++) { + if (schema_[i] != types[permutation_[i]]) + return errors::Unknown("Object has unexpected schema"); + } + + return Status::OK(); +} + +int32_t IgniteDatasetIterator::JavaHashCode(string str) const { int32_t h = 0; for (char& c : str) { h = 31 * h + c; diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h index 5858dbfcb9..c499e2c9cc 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_iterator.h @@ -13,19 +13,22 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_binary_object_parser.h" -#include "ignite_client.h" -#include "ignite_dataset.h" +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_ITERATOR_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_ITERATOR_H_ + +#include "tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h" +#include "tensorflow/contrib/ignite/kernels/ignite_client.h" +#include "tensorflow/contrib/ignite/kernels/ignite_dataset.h" +#include "tensorflow/core/platform/mutex.h" namespace tensorflow { class IgniteDatasetIterator : public DatasetIterator { public: - IgniteDatasetIterator(const Params& params, std::string host, int32 port, - std::string cache_name, bool local, int32 part, - int32 page_size, std::string username, - std::string password, std::string certfile, - std::string keyfile, std::string cert_password, + IgniteDatasetIterator(const Params& params, string host, int32 port, + string cache_name, bool local, int32 part, + int32 page_size, string username, string password, + string certfile, string keyfile, string cert_password, std::vector schema, std::vector permutation); ~IgniteDatasetIterator(); @@ -38,15 +41,28 @@ class IgniteDatasetIterator : public DatasetIterator { IteratorStateReader* reader) override; private: + Status GetNextInternalWithValidState(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence); + + Status EstablishConnection(); + Status CloseConnection(); + Status Handshake(); + Status ScanQuery(); + Status LoadNextPage(); + Status ReceivePage(int32_t page_size); + Status CheckTypes(const std::vector& types); + int32_t JavaHashCode(string str) const; + std::unique_ptr client_; BinaryObjectParser parser_; - const std::string cache_name_; + const string cache_name_; const bool local_; const int32 part_; const int32 page_size_; - const std::string username_; - const std::string password_; + const string username_; + const string password_; const std::vector schema_; const std::vector permutation_; @@ -54,24 +70,30 @@ class IgniteDatasetIterator : public DatasetIterator { int64_t cursor_id_; bool last_page_; + bool valid_state_; + + mutex mutex_; + std::unique_ptr page_; uint8_t* ptr_; - - Status EstablishConnection(); - Status CloseConnection(); - Status Handshake(); - Status ScanQuery(); - Status LoadNextPage(); - int32_t JavaHashCode(std::string str) const; }; -constexpr uint8_t null_val = 101; -constexpr uint8_t string_val = 9; -constexpr uint8_t protocol_major_version = 1; -constexpr uint8_t protocol_minor_version = 1; -constexpr uint8_t protocol_patch_version = 0; -constexpr int16_t scan_query_opcode = 2000; -constexpr int16_t load_next_page_opcode = 2001; -constexpr int16_t close_connection_opcode = 0; +constexpr uint8_t kNullVal = 101; +constexpr uint8_t kStringVal = 9; +constexpr uint8_t kProtocolMajorVersion = 1; +constexpr uint8_t kProtocolMinorVersion = 1; +constexpr uint8_t kProtocolPatchVersion = 0; +constexpr int16_t kScanQueryOpcode = 2000; +constexpr int16_t kLoadNextPageOpcode = 2001; +constexpr int16_t kCloseConnectionOpcode = 0; +constexpr int32_t kScanQueryReqLength = 25; +constexpr int32_t kScanQueryResHeaderLength = 25; +constexpr int32_t kLoadNextPageReqLength = 18; +constexpr int32_t kLoadNextPageResHeaderLength = 17; +constexpr int32_t kCloseConnectionReqLength = 18; +constexpr int32_t kHandshakeReqDefaultLength = 8; +constexpr int32_t kMinResLength = 12; } // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_DATASET_ITERATOR_H_ diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc index d03404a460..eeb29ef30b 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -13,29 +13,73 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/contrib/ignite/kernels/ignite_dataset.h" #include -#include "ignite_dataset.h" +#include "tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h" #include "tensorflow/core/framework/dataset.h" namespace tensorflow { namespace { +Status SchemaToTypes(const std::vector& schema, DataTypeVector* dtypes) { + for (auto e : schema) { + if (e == BYTE || e == BYTE_ARR) { + dtypes->push_back(DT_UINT8); + } else if (e == SHORT || e == SHORT_ARR) { + dtypes->push_back(DT_INT16); + } else if (e == INT || e == INT_ARR) { + dtypes->push_back(DT_INT32); + } else if (e == LONG || e == LONG_ARR) { + dtypes->push_back(DT_INT64); + } else if (e == FLOAT || e == FLOAT_ARR) { + dtypes->push_back(DT_FLOAT); + } else if (e == DOUBLE || e == DOUBLE_ARR) { + dtypes->push_back(DT_DOUBLE); + } else if (e == USHORT || e == USHORT_ARR) { + dtypes->push_back(DT_UINT8); + } else if (e == BOOL || e == BOOL_ARR) { + dtypes->push_back(DT_BOOL); + } else if (e == STRING || e == STRING_ARR) { + dtypes->push_back(DT_STRING); + } else { + return errors::Unknown("Unexpected type in schema [type_id=", e, "]"); + } + } + + return Status::OK(); +} + +Status SchemaToShapes(const std::vector& schema, + std::vector* shapes) { + for (auto e : schema) { + if (e >= 1 && e < 10) { + shapes->push_back(PartialTensorShape({})); + } else if (e >= 12 && e < 21) { + shapes->push_back(PartialTensorShape({-1})); + } else { + return errors::Unknown("Unexpected type in schema [type_id=", e, "]"); + } + } + + return Status::OK(); +} + class IgniteDatasetOp : public DatasetOpKernel { public: using DatasetOpKernel::DatasetOpKernel; void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - std::string cache_name = ""; - std::string host = ""; + string cache_name = ""; + string host = ""; int32 port = -1; bool local = false; int32 part = -1; int32 page_size = -1; - std::string username = ""; - std::string password = ""; - std::string certfile = ""; - std::string keyfile = ""; - std::string cert_password = ""; + string username = ""; + string password = ""; + string certfile = ""; + string keyfile = ""; + string cert_password = ""; const char* env_cache_name = std::getenv("IGNITE_DATASET_CACHE_NAME"); const char* env_host = std::getenv("IGNITE_DATASET_HOST"); @@ -50,15 +94,15 @@ class IgniteDatasetOp : public DatasetOpKernel { const char* env_cert_password = std::getenv("IGNITE_DATASET_CERT_PASSWORD"); if (env_cache_name) - cache_name = std::string(env_cache_name); + cache_name = string(env_cache_name); else - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cache_name", - &cache_name)); + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "cache_name", &cache_name)); if (env_host) - host = std::string(env_host); + host = string(env_host); else - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "host", &host)); + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "host", &host)); if (env_port) port = atoi(env_port); @@ -82,34 +126,34 @@ class IgniteDatasetOp : public DatasetOpKernel { ParseScalarArgument(ctx, "page_size", &page_size)); if (env_username) - username = std::string(env_username); + username = string(env_username); else - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "username", &username)); + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, "username", &username)); if (env_password) - password = std::string(env_password); + password = string(env_password); else - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "password", &password)); + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, "password", &password)); if (env_certfile) - certfile = std::string(env_certfile); + certfile = string(env_certfile); else - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "certfile", &certfile)); + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, "certfile", &certfile)); if (env_keyfile) - keyfile = std::string(env_keyfile); + keyfile = string(env_keyfile); else - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "keyfile", &keyfile)); + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, "keyfile", &keyfile)); if (env_cert_password) - cert_password = std::string(env_cert_password); + cert_password = string(env_cert_password); else - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cert_password", - &cert_password)); + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cert_password", + &cert_password)); const Tensor* schema_tensor; OP_REQUIRES_OK(ctx, ctx->input("schema", &schema_tensor)); @@ -124,19 +168,28 @@ class IgniteDatasetOp : public DatasetOpKernel { const Tensor* permutation_tensor; OP_REQUIRES_OK(ctx, ctx->input("permutation", &permutation_tensor)); - OP_REQUIRES(ctx, schema_tensor->dims() == 1, + OP_REQUIRES(ctx, permutation_tensor->dims() == 1, errors::InvalidArgument("`permutation` must be a vector.")); std::vector permutation; - permutation.reserve(permutation_tensor->NumElements()); + permutation.resize(permutation_tensor->NumElements()); for (int i = 0; i < permutation_tensor->NumElements(); i++) { - permutation.push_back(permutation_tensor->flat()(i)); + // Inversed permutation. + permutation[permutation_tensor->flat()(i)] = i; } - *output = - new IgniteDataset(ctx, cache_name, host, port, local, part, page_size, - username, password, certfile, keyfile, cert_password, - std::move(schema), std::move(permutation)); + DataTypeVector dtypes; + std::vector shapes; + + OP_REQUIRES_OK(ctx, SchemaToTypes(schema, &dtypes)); + OP_REQUIRES_OK(ctx, SchemaToShapes(schema, &shapes)); + + *output = new IgniteDataset( + ctx, std::move(cache_name), std::move(host), port, local, part, + page_size, std::move(username), std::move(password), + std::move(certfile), std::move(keyfile), std::move(cert_password), + std::move(schema), std::move(permutation), std::move(dtypes), + std::move(shapes)); } }; diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h index 6f417a3cb5..750ebe605a 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h @@ -13,28 +13,31 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_client.h" +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_PLAIN_CLIENT_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_PLAIN_CLIENT_H_ -#include +#include "tensorflow/contrib/ignite/kernels/ignite_client.h" namespace tensorflow { class PlainClient : public Client { public: - PlainClient(std::string host, int port); + PlainClient(string host, int port, bool big_endian); ~PlainClient(); virtual Status Connect(); virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual Status ReadData(uint8_t* buf, int32_t length); - virtual Status WriteData(uint8_t* buf, int32_t length); + virtual Status ReadData(uint8_t* buf, const int32_t length); + virtual Status WriteData(const uint8_t* buf, const int32_t length); private: - const std::string host_; + const string host_; const int port_; int sock_; }; } // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_PLAIN_CLIENT_H_ diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc index a4c58a9563..e16c92307d 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_unix.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_plain_client.h" +#include "tensorflow/contrib/ignite/kernels/ignite_plain_client.h" #include #include @@ -31,8 +31,8 @@ limitations under the License. namespace tensorflow { -PlainClient::PlainClient(std::string host, int port) - : host_(host), port_(port), sock_(-1) {} +PlainClient::PlainClient(string host, int port, bool big_endian) + : Client(big_endian), host_(std::move(host)), port_(port), sock_(-1) {} PlainClient::~PlainClient() { if (IsConnected()) { @@ -87,7 +87,7 @@ bool PlainClient::IsConnected() { return sock_ != -1; } int PlainClient::GetSocketDescriptor() { return sock_; } -Status PlainClient::ReadData(uint8_t* buf, int32_t length) { +Status PlainClient::ReadData(uint8_t* buf, const int32_t length) { int recieved = 0; while (recieved < length) { @@ -95,7 +95,7 @@ Status PlainClient::ReadData(uint8_t* buf, int32_t length) { if (res < 0) return errors::Internal("Error occured while reading from socket: ", res, - ", ", std::string(strerror(errno))); + ", ", string(strerror(errno))); if (res == 0) return errors::Internal("Server closed connection"); @@ -106,7 +106,7 @@ Status PlainClient::ReadData(uint8_t* buf, int32_t length) { return Status::OK(); } -Status PlainClient::WriteData(uint8_t* buf, int32_t length) { +Status PlainClient::WriteData(const uint8_t* buf, const int32_t length) { int sent = 0; while (sent < length) { @@ -114,7 +114,7 @@ Status PlainClient::WriteData(uint8_t* buf, int32_t length) { if (res < 0) return errors::Internal("Error occured while writing into socket: ", res, - ", ", std::string(strerror(errno))); + ", ", string(strerror(errno))); sent += res; buf += res; diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index 8182fde6d9..9cd08a7779 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_plain_client.h" +#include "tensorflow/contrib/ignite/kernels/ignite_plain_client.h" #define WIN32_LEAN_AND_MEAN #include @@ -29,8 +29,11 @@ limitations under the License. namespace tensorflow { -PlainClient::PlainClient(std::string host, int port) - : host_(host), port_(port), sock_(INVALID_SOCKET) {} +PlainClient::PlainClient(string host, int port, bool big_endian) + : Client(big_endian), + host_(std::move(host)), + port_(port), + sock_(INVALID_SOCKET) {} PlainClient::~PlainClient() { if (IsConnected()) { @@ -55,6 +58,8 @@ Status PlainClient::Connect() { &result); if (res != 0) return errors::Internal("Getaddrinfo failed with error: ", res); + auto clean = gtl::MakeCleanup([result] { reeaddrinfo(result); }); + for (ptr = result; ptr != NULL; ptr = ptr->ai_next) { sock_ = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); if (sock_ == INVALID_SOCKET) { @@ -72,8 +77,6 @@ Status PlainClient::Connect() { break; } - freeaddrinfo(result); - if (sock_ == INVALID_SOCKET) { WSACleanup(); return errors::Internal("Unable to connect to server"); @@ -99,7 +102,7 @@ bool PlainClient::IsConnected() { return sock_ != INVALID_SOCKET; } int PlainClient::GetSocketDescriptor() { return sock_; } -Status PlainClient::ReadData(uint8_t *buf, int32_t length) { +Status PlainClient::ReadData(uint8_t *buf, const int32_t length) { int recieved = 0; while (recieved < length) { @@ -117,7 +120,7 @@ Status PlainClient::ReadData(uint8_t *buf, int32_t length) { return Status::OK(); } -Status PlainClient::WriteData(uint8_t *buf, int32_t length) { +Status PlainClient::WriteData(const uint8_t *buf, const int32_t length) { int sent = 0; while (sent < length) { diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc index a2bc6b9609..28db509eaa 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_ssl_wrapper.h" +#include "tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" @@ -29,13 +29,15 @@ static int PasswordCb(char *buf, int size, int rwflag, void *password) { return (strlen(buf)); } -SslWrapper::SslWrapper(std::shared_ptr client, std::string certfile, - std::string keyfile, std::string cert_password) - : client_(client), - certfile_(certfile), - keyfile_(keyfile), - cert_password_(cert_password), - ctx_(NULL) {} +SslWrapper::SslWrapper(std::shared_ptr client, string certfile, + string keyfile, string cert_password, bool big_endian) + : Client(big_endian), + client_(client), + certfile_(std::move(certfile)), + keyfile_(std::move(keyfile)), + cert_password_(std::move(cert_password)), + ctx_(nullptr), + ssl_(nullptr) {} SslWrapper::~SslWrapper() { if (IsConnected()) { @@ -43,9 +45,14 @@ SslWrapper::~SslWrapper() { if (!status.ok()) LOG(WARNING) << status.ToString(); } - if (ctx_ != NULL) { + if (ctx_ != nullptr) { SSL_CTX_free(ctx_); - ctx_ = NULL; + ctx_ = nullptr; + } + + if (ssl_ != nullptr) { + SSL_free(ssl_); + ssl_ = nullptr; } } @@ -63,7 +70,7 @@ Status SslWrapper::InitSslContext() { return errors::Internal("Couldn't load cetificate chain (file '", certfile_, "')"); - std::string private_key_file = keyfile_.empty() ? certfile_ : keyfile_; + string private_key_file = keyfile_.empty() ? certfile_ : keyfile_; if (SSL_CTX_use_PrivateKey_file(ctx_, private_key_file.c_str(), SSL_FILETYPE_PEM) != 1) return errors::Internal("Couldn't load private key (file '", @@ -94,6 +101,7 @@ Status SslWrapper::Connect() { Status SslWrapper::Disconnect() { SSL_free(ssl_); + ssl_ = nullptr; LOG(INFO) << "SSL connection closed"; @@ -104,7 +112,7 @@ bool SslWrapper::IsConnected() { return client_->IsConnected(); } int SslWrapper::GetSocketDescriptor() { return client_->GetSocketDescriptor(); } -Status SslWrapper::ReadData(uint8_t *buf, int32_t length) { +Status SslWrapper::ReadData(uint8_t *buf, const int32_t length) { int recieved = 0; while (recieved < length) { @@ -123,7 +131,7 @@ Status SslWrapper::ReadData(uint8_t *buf, int32_t length) { return Status::OK(); } -Status SslWrapper::WriteData(uint8_t *buf, int32_t length) { +Status SslWrapper::WriteData(const uint8_t *buf, const int32_t length) { int sent = 0; while (sent < length) { diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h index bbba6cc181..d59ce91aba 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h @@ -13,35 +13,39 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "ignite_client.h" +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_SSL_WRAPPER_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_SSL_WRAPPER_H_ + +#include "tensorflow/contrib/ignite/kernels/ignite_client.h" #include -#include namespace tensorflow { class SslWrapper : public Client { public: - SslWrapper(std::shared_ptr client, std::string certfile, - std::string keyfile, std::string cert_password); + SslWrapper(std::shared_ptr client, string certfile, string keyfile, + string cert_password, bool big_endian); ~SslWrapper(); virtual Status Connect(); virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual Status ReadData(uint8_t* buf, int32_t length); - virtual Status WriteData(uint8_t* buf, int32_t length); + virtual Status ReadData(uint8_t* buf, const int32_t length); + virtual Status WriteData(const uint8_t* buf, const int32_t length); private: + Status InitSslContext(); + std::shared_ptr client_; - std::string certfile_; - std::string keyfile_; - std::string cert_password_; + string certfile_; + string keyfile_; + string cert_password_; SSL_CTX* ctx_; SSL* ssl_; - - Status InitSslContext(); }; } // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_SSL_WRAPPER_H_ \ No newline at end of file diff --git a/tensorflow/contrib/ignite/ops/dataset_ops.cc b/tensorflow/contrib/ignite/ops/dataset_ops.cc index fb16b290b1..7d18df11aa 100644 --- a/tensorflow/contrib/ignite/ops/dataset_ops.cc +++ b/tensorflow/contrib/ignite/ops/dataset_ops.cc @@ -37,6 +37,8 @@ REGISTER_OP("IgniteDataset") .SetIsStateful() .SetShapeFn(shape_inference::ScalarShape) .Doc(R"doc( +IgniteDataset that allows to get data from Apache Ignite. + Apache Ignite is a memory-centric distributed database, caching, and processing platform for transactional, analytical, and streaming workloads, delivering in-memory speeds at petabyte scale. This contrib package contains an diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py index 60003ca3b7..c0e24b1c69 100644 --- a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py +++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py @@ -41,19 +41,19 @@ class Readable(): def read_byte(self): """Reads and returnes byte.""" - return self.__read("b", 1) + return self._read("b", 1) def read_short(self): """Reads and returns short (2 bytes, little-endian).""" - return self.__read("h", 2) + return self._read("h", 2) def read_int(self): """Reads and returns int (4 bytes, little-endian).""" - return self.__read("i", 4) + return self._read("i", 4) def read_long(self): """Reads and returns long (8 bytes, little-endian).""" - return self.__read("q", 8) + return self._read("q", 8) def skip(self, length): """Skips the specified number of bytes.""" @@ -64,7 +64,7 @@ class Readable(): """Reads the specified number of bytes and returns them as a buffer.""" return None - def __read(self, data_type, length): + def _read(self, data_type, length): """Reads, unpacks and returns specified type (little-endian).""" data_buffer = self.read_data(length) return struct.unpack("<" + data_type, data_buffer)[0] @@ -116,10 +116,10 @@ class TcpClient(Readable): self.sock = context.wrap_socket(self.sock) else: if keyfile is not None: - raise Exception("SSL is disabled, keyfile must not be specified \ + raise RuntimeError("SSL is disabled, keyfile must not be specified \ (to enable SSL specify certfile)") if password is not None: - raise Exception("SSL is disabled, password must not be specified \ + raise RuntimeError("SSL is disabled, password must not be specified \ (to enable SSL specify certfile)") self.host = host @@ -136,19 +136,19 @@ class TcpClient(Readable): def write_byte(self, v): """Writes the specified byte.""" - self.__write(v, "b") + self._write(v, "b") def write_short(self, v): """Writes the specified short (2 bytes, little-endian).""" - self.__write(v, "h") + self._write(v, "h") def write_int(self, v): """Writes the specified short (4 bytes, little-endian).""" - self.__write(v, "i") + self._write(v, "i") def write_long(self, v): """Writes the specified int (8 bytes, little-endian).""" - self.__write(v, "q") + self._write(v, "q") def write_string(self, v): """Writes the specified string.""" @@ -167,7 +167,7 @@ class TcpClient(Readable): data_buffer += buf return data_buffer - def __write(self, value, data_type): + def _write(self, value, data_type): """Packs and writes data using the specified type (little-endian).""" data_buffer = struct.pack("<" + data_type, value) self.sock.sendall(data_buffer) @@ -193,6 +193,7 @@ class BinaryField(): # Binary types defined in Apache Ignite Thin client and supported by # TensorFlow on Apache Ignite, see # https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. +# True means that type is a vector, False means type is scalar. types = { 1: (dtypes.uint8, False), 2: (dtypes.int16, False), @@ -248,13 +249,13 @@ class TypeTreeNode(): dataset. """ if self.fields is None: - object_type = types[self.type_id] - if object_type is not None: + if self.type_id in types: + object_type = types[self.type_id] is_array = object_type[1] if is_array: return tensor_shape.TensorShape([None]) return tensor_shape.TensorShape([]) - raise Exception("Unsupported type [type_id=%d]" % self.type_id) + raise ValueError("Unsupported type [type_id=%d]" % self.type_id) output_shapes = {} for field in self.fields: output_shapes[field.name] = field.to_output_shapes() @@ -265,10 +266,10 @@ class TypeTreeNode(): dataset. """ if self.fields is None: - object_type = types[self.type_id] - if object_type is not None: + if self.type_id in types: + object_type = types[self.type_id] return object_type[0] - raise Exception("Unsupported type [type_id=%d]" % self.type_id) + raise ValueError("Unsupported type [type_id=%d]" % self.type_id) else: output_types = {} for field in self.fields: @@ -276,11 +277,11 @@ class TypeTreeNode(): return output_types def to_flat(self): - """Returns a list of leaf node types.""" + """Returns a list of node types.""" return self.to_flat_rec([]) def to_permutation(self): - """Returns a permutation that should be applied to order object leafs.""" + """Returns a permutation that should be applied to order object leaves.""" correct_order_dict = {} self.traversal_rec(correct_order_dict, 0) object_order = [] @@ -288,9 +289,10 @@ class TypeTreeNode(): return [correct_order_dict[o] for o in object_order] def to_flat_rec(self, flat): - """Formats a list of leaf node types.""" - flat.append(self.type_id) - if self.fields is not None: + """Formats a list of leaf node types in pre-order.""" + if self.fields is None: + flat.append(self.type_id) + else: for field in self.fields: field.to_flat_rec(flat) return flat @@ -320,8 +322,8 @@ class IgniteClient(TcpClient): have the same structure (homogeneous objects) and the cache contains at least one object. """ - def __init__(self, host, port, username=None, password=None, certfile=None,\ - keyfile=None, cert_password=None): + def __init__(self, host, port, username=None, password=None, certfile=None, + keyfile=None, cert_password=None): """Constructs a new instance of IgniteClient. Args: @@ -385,12 +387,13 @@ class IgniteClient(TcpClient): serv_ver_major = self.read_short() serv_ver_minor = self.read_short() serv_ver_patch = self.read_short() - err_msg = self.__parse_string() + err_msg = self._parse_string() if err_msg is None: - raise Exception("Handshake Error [result=%d, version=%d.%d.%d]" \ - % (res, serv_ver_major, serv_ver_minor, serv_ver_patch)) + raise RuntimeError("Handshake Error [result=%d, version=%d.%d.%d]" + % (res, serv_ver_major, serv_ver_minor, + serv_ver_patch)) else: - raise Exception("Handshake Error [result=%d, version=%d.%d.%d, \ + raise RuntimeError("Handshake Error [result=%d, version=%d.%d.%d, \ message='%s']" % ( res, serv_ver_major, @@ -403,7 +406,7 @@ class IgniteClient(TcpClient): """Collects type information about objects stored in the specified cache. """ - cache_name_hash = self.__java_hash_code(cache_name) + cache_name_hash = self._java_hash_code(cache_name) self.write_int(25) # Message length self.write_short(2000) # Operation code self.write_long(0) # Request ID @@ -419,18 +422,18 @@ class IgniteClient(TcpClient): status = self.read_int() if status != 0: - err_msg = self.__parse_string() + err_msg = self._parse_string() if err_msg is None: - raise Exception("Scan Query Error [status=%s]" % status) + raise RuntimeError("Scan Query Error [status=%s]" % status) else: - raise Exception("Scan Query Error [status=%s, message='%s']" \ - % (status, err_msg)) + raise RuntimeError("Scan Query Error [status=%s, message='%s']" + % (status, err_msg)) self.read_long() # Cursor id row_count = self.read_int() if row_count == 0: - raise Exception("Scan Query returned empty result, so it's \ + raise RuntimeError("Scan Query returned empty result, so it's \ impossible to derive the cache type") payload = DataBuffer(self.read_data(result_length - 25)) @@ -438,20 +441,20 @@ class IgniteClient(TcpClient): self.read_byte() # Next page res = TypeTreeNode("root", 0, [ - self.__collect_types("key", payload), - self.__collect_types("val", payload) + self._collect_types("key", payload), + self._collect_types("val", payload) ], [0, 1]) return res - def __java_hash_code(self, s): + def _java_hash_code(self, s): """Computes hash code of the specified string using Java code.""" h = 0 for c in s: h = (31 * h + ord(c)) & 0xFFFFFFFF return ((h + 0x80000000) & 0xFFFFFFFF) - 0x80000000 - def __collect_types(self, field_name, data): + def _collect_types(self, field_name, data): """Extracts type information from the specified object.""" type_id = data.read_byte() @@ -570,7 +573,7 @@ class IgniteClient(TcpClient): elif header == 101: pass else: - raise Exception("Unknown binary type when expected string \ + raise RuntimeError("Unknown binary type when expected string \ [type_id=%d]" % header) return TypeTreeNode(field_name, type_id) @@ -591,7 +594,7 @@ class IgniteClient(TcpClient): length = data.read_int() inner_data = data.read_data(length) data.read_int() # Offset - return self.__collect_types(field_name, DataBuffer(inner_data)) + return self._collect_types(field_name, DataBuffer(inner_data)) # Complex Object. if type_id == 103: @@ -603,11 +606,11 @@ class IgniteClient(TcpClient): data.read_int() # Object schema id obj_schema_offset = data.read_int() - obj_type = self.__get_type(obj_type_id) + obj_type = self._get_type(obj_type_id) children = [] for obj_field in obj_type.fields: - child = self.__collect_types(obj_field.field_name, data) + child = self._collect_types(obj_field.field_name, data) children.append(child) children_sorted = sorted(children, key=lambda child: child.name) @@ -618,9 +621,9 @@ class IgniteClient(TcpClient): return TypeTreeNode(field_name, type_id, children, permutation) - raise Exception("Unknown binary type [type_id=%d]" % type_id) + raise RuntimeError("Unknown binary type [type_id=%d]" % type_id) - def __get_type(self, type_id): + def _get_type(self, type_id): """Queries Apache Ignite information about type by type id.""" self.write_int(14) # Message length self.write_short(3002) # Operation code @@ -632,25 +635,25 @@ class IgniteClient(TcpClient): status = self.read_int() if status != 0: - err_msg = self.__parse_string() + err_msg = self._parse_string() if err_msg is None: - raise Exception("Get Binary Type Error [status=%d, message='%s']" \ - % (status, err_msg)) + raise RuntimeError("Get Binary Type Error [status=%d, message='%s']" + % (status, err_msg)) else: - raise Exception("Get Binary Type Error [status=%d]" % status) + raise RuntimeError("Get Binary Type Error [status=%d]" % status) binary_type_exists = self.read_byte() if binary_type_exists == 0: - raise Exception("Binary type not found [type_id=%d] " % type_id) + raise RuntimeError("Binary type not found [type_id=%d] " % type_id) binary_type_id = self.read_int() - binary_type_name = self.__parse_string() - self.__parse_string() # Affinity field name + binary_type_name = self._parse_string() + self._parse_string() # Affinity field name fields = [] for _ in range(self.read_int()): - field_name = self.__parse_string() + field_name = self._parse_string() field_type_id = self.read_int() field_id = self.read_int() @@ -659,7 +662,7 @@ class IgniteClient(TcpClient): is_enum = self.read_byte() if is_enum == 1: - raise Exception("Enum fields are not supported yet") + raise RuntimeError("Enum fields are not supported yet") schema_cnt = self.read_int() for _ in range(schema_cnt): @@ -669,7 +672,7 @@ class IgniteClient(TcpClient): return BinaryType(binary_type_id, binary_type_name, fields) - def __parse_string(self): + def _parse_string(self): """Parses string.""" header = self.read_byte() if header == 9: @@ -677,8 +680,8 @@ class IgniteClient(TcpClient): return self.read_data(length).decode("utf-8") if header == 101: return None - raise Exception("Unknown binary type when expected string [type_id=%d]" \ - % header) + raise RuntimeError("Unknown binary type when expected string [type_id=%d]" + % header) class IgniteDataset(Dataset): """Apache Ignite is a memory-centric distributed database, caching, and @@ -692,9 +695,9 @@ class IgniteDataset(Dataset): Ignite Binary Client Protocol. """ - def __init__(self, cache_name, host="localhost", port=10800, local=False,\ - part=-1, page_size=100, username=None, password=None, certfile=None,\ - keyfile=None, cert_password=None): + def __init__(self, cache_name, host="localhost", port=10800, local=False, + part=-1, page_size=100, username=None, password=None, + certfile=None, keyfile=None, cert_password=None): """Create a IgniteDataset. Args: @@ -716,39 +719,44 @@ class IgniteDataset(Dataset): """ super(IgniteDataset, self).__init__() - with IgniteClient(host, port, username, password, certfile, keyfile,\ - cert_password) as client: + with IgniteClient(host, port, username, password, certfile, keyfile, + cert_password) as client: client.handshake() self.cache_type = client.get_cache_type(cache_name) - self.cache_name = ops.convert_to_tensor(cache_name, dtype=dtypes.string,\ - name="cache_name") + self.cache_name = ops.convert_to_tensor(cache_name, dtype=dtypes.string, + name="cache_name") self.host = ops.convert_to_tensor(host, dtype=dtypes.string, name="host") self.port = ops.convert_to_tensor(port, dtype=dtypes.int32, name="port") self.local = ops.convert_to_tensor(local, dtype=dtypes.bool, name="local") self.part = ops.convert_to_tensor(part, dtype=dtypes.int32, name="part") - self.page_size = ops.convert_to_tensor(page_size, dtype=dtypes.int32,\ - name="page_size") - self.username = ops.convert_to_tensor("" if username is None else username,\ - dtype=dtypes.string, name="username") - self.password = ops.convert_to_tensor("" if password is None else password,\ - dtype=dtypes.string, name="password") - self.certfile = ops.convert_to_tensor("" if certfile is None else certfile,\ - dtype=dtypes.string, name="certfile") - self.keyfile = ops.convert_to_tensor("" if keyfile is None else keyfile,\ - dtype=dtypes.string, name="keyfile") - self.cert_password = ops.convert_to_tensor("" if cert_password is None\ - else cert_password, dtype=dtypes.string, name="cert_password") - self.schema = ops.convert_to_tensor(self.cache_type.to_flat(),\ - dtype=dtypes.int32, name="schema") - self.permutation = ops.convert_to_tensor(self.cache_type.to_permutation(),\ - dtype=dtypes.int32, name="permutation") + self.page_size = ops.convert_to_tensor(page_size, dtype=dtypes.int32, + name="page_size") + self.username = ops.convert_to_tensor("" if username is None else username, + dtype=dtypes.string, name="username") + self.password = ops.convert_to_tensor("" if password is None else password, + dtype=dtypes.string, name="password") + self.certfile = ops.convert_to_tensor("" if certfile is None else certfile, + dtype=dtypes.string, name="certfile") + self.keyfile = ops.convert_to_tensor("" if keyfile is None else keyfile, + dtype=dtypes.string, name="keyfile") + self.cert_password = ops.convert_to_tensor("" if cert_password is None + else cert_password, + dtype=dtypes.string, + name="cert_password") + self.schema = ops.convert_to_tensor(self.cache_type.to_flat(), + dtype=dtypes.int32, name="schema") + self.permutation = ops.convert_to_tensor(self.cache_type.to_permutation(), + dtype=dtypes.int32, + name="permutation") def _as_variant_tensor(self): - return gen_dataset_ops.ignite_dataset(self.cache_name, self.host,\ - self.port, self.local, self.part, self.page_size, self.username,\ - self.password, self.certfile, self.keyfile, self.cert_password,\ - self.schema, self.permutation) + return gen_dataset_ops.ignite_dataset(self.cache_name, self.host, + self.port, self.local, self.part, + self.page_size, self.username, + self.password, self.certfile, + self.keyfile, self.cert_password, + self.schema, self.permutation) @property def output_classes(self): -- GitLab From 172d199db9bab808723d24ea586322f1b2d80413 Mon Sep 17 00:00:00 2001 From: leondgarse Date: Thu, 13 Sep 2018 16:12:18 +0800 Subject: [PATCH 0052/1085] Use the last char instead of the first in prediction --- .../examples/generative_examples/text_generation.ipynb | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb index e0d5e494d4..07dbfd3630 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb @@ -598,19 +598,13 @@ "# empty string to store our results\n", "text_generated = ''\n", "\n", - "# low temperatures results in more predictable text.\n", - "# higher temperatures results in more surprising text\n", - "# experiment to find the best setting\n", - "temperature = 1.0\n", - "\n", "# hidden state shape == (batch_size, number of rnn units); here batch size == 1\n", "hidden = [tf.zeros((1, units))]\n", "for i in range(num_generate):\n", " predictions, hidden = model(input_eval, hidden)\n", "\n", " # using a multinomial distribution to predict the word returned by the model\n", - " predictions = predictions / temperature\n", - " predicted_id = tf.argmax(predictions[0]).numpy()\n", + " predicted_id = tf.argmax(predictions[-1]).numpy()\n", " \n", " # We pass the predicted word as the next input to the model\n", " # along with the previous hidden state\n", -- GitLab From ce9b23070638094022036656e5d1fbf3e23b74c6 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Thu, 13 Sep 2018 11:24:37 +0300 Subject: [PATCH 0053/1085] Add forgotten ignite_byte_swapper.h --- .../ignite/kernels/ignite_byte_swapper.h | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h diff --git a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h new file mode 100644 index 0000000000..986bedcf69 --- /dev/null +++ b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h @@ -0,0 +1,129 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BYTE_SWAPPER_H_ +#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BYTE_SWAPPER_H_ + +#include + +namespace tensorflow { + +class ByteSwapper { + public: + ByteSwapper(bool big_endian) { + int x = 1; + bool is_little_endian = (*(char *)&x == 1); + swap_ = big_endian == is_little_endian; + } + + inline void SwapIfRequiredInt16(int16_t *x) const { + if (swap_) { + Swap16(x); + } + } + + inline void SwapIfRequiredUnsignedInt16(uint16_t *x) const { + if (swap_) { + Swap16(reinterpret_cast(x)); + } + } + + inline void SwapIfRequiredInt32(int32_t *x) const { + if (swap_) { + Swap32(x); + } + } + + inline void SwapIfRequiredFloat(float *x) const { + if (swap_) { + Swap32(reinterpret_cast(x)); + } + } + + inline void SwapIfRequiredInt64(int64_t *x) const { + if (swap_) { + Swap64(x); + } + } + + inline void SwapIfRequiredDouble(double *x) const { + if (swap_) { + Swap64(reinterpret_cast(x)); + } + } + + inline void SwapIfRequiredInt16Arr(int16_t *x, int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) Swap16(&x[i]); + } + } + + inline void SwapIfRequiredUnsignedInt16Arr(uint16_t *x, + int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) + Swap16(reinterpret_cast(&x[i])); + } + } + + inline void SwapIfRequiredInt32Arr(int32_t *x, int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) Swap32(&x[i]); + } + } + + inline void SwapIfRequiredFloatArr(float *x, int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) + Swap32(reinterpret_cast(&x[i])); + } + } + + inline void SwapIfRequiredInt64Arr(int64_t *x, int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) Swap64(&x[i]); + } + } + + inline void SwapIfRequiredDoubleArr(double *x, int32_t length) const { + if (swap_) { + for (int32_t i = 0; i < length; i++) + Swap64(reinterpret_cast(&x[i])); + } + } + + private: + inline void Swap16(int16_t *x) const { + *x = ((*x & 0xFF) << 8) | ((*x >> 8) & 0xFF); + } + + inline void Swap32(int32_t *x) const { + *x = ((*x & 0xFF) << 24) | (((*x >> 8) & 0xFF) << 16) | + (((*x >> 16) & 0xFF) << 8) | ((*x >> 24) & 0xFF); + } + + inline void Swap64(int64_t *x) const { + *x = ((*x & 0xFF) << 56) | (((*x >> 8) & 0xFF) << 48) | + (((*x >> 16) & 0xFF) << 40) | (((*x >> 24) & 0xFF) << 32) | + (((*x >> 32) & 0xFF) << 24) | (((*x >> 40) & 0xFF) << 16) | + (((*x >> 48) & 0xFF) << 8) | ((*x >> 56) & 0xFF); + } + + bool swap_; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BYTE_SWAPPER_H_ -- GitLab From d797e99a043e01609583a37c04e1e509d126e1a0 Mon Sep 17 00:00:00 2001 From: dmitrievanthony Date: Thu, 13 Sep 2018 09:42:16 +0000 Subject: [PATCH 0054/1085] Fix windows build. --- .../contrib/ignite/kernels/ignite_plain_client_windows.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index 9cd08a7779..17f2bf45d1 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -24,6 +24,7 @@ limitations under the License. #pragma comment(lib, "Mswsock.lib") #pragma comment(lib, "AdvApi32.lib") +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" @@ -58,7 +59,7 @@ Status PlainClient::Connect() { &result); if (res != 0) return errors::Internal("Getaddrinfo failed with error: ", res); - auto clean = gtl::MakeCleanup([result] { reeaddrinfo(result); }); + auto clean = gtl::MakeCleanup([result] { freeaddrinfo(result); }); for (ptr = result; ptr != NULL; ptr = ptr->ai_next) { sock_ = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); -- GitLab From c8b60b894b91cfdb4176176d7dcf328d2b40b41f Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Thu, 13 Sep 2018 16:34:59 +0300 Subject: [PATCH 0055/1085] Fix code style. --- .../ignite/kernels/ignite_byte_swapper.h | 18 +++++++++--------- .../ignite/kernels/ignite_dataset_ops.cc | 2 +- .../kernels/ignite_plain_client_windows.cc | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h index 986bedcf69..5b42de4c5a 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h @@ -36,7 +36,7 @@ class ByteSwapper { inline void SwapIfRequiredUnsignedInt16(uint16_t *x) const { if (swap_) { - Swap16(reinterpret_cast(x)); + Swap16(reinterpret_cast(x)); } } @@ -48,7 +48,7 @@ class ByteSwapper { inline void SwapIfRequiredFloat(float *x) const { if (swap_) { - Swap32(reinterpret_cast(x)); + Swap32(reinterpret_cast(x)); } } @@ -60,7 +60,7 @@ class ByteSwapper { inline void SwapIfRequiredDouble(double *x) const { if (swap_) { - Swap64(reinterpret_cast(x)); + Swap64(reinterpret_cast(x)); } } @@ -73,8 +73,8 @@ class ByteSwapper { inline void SwapIfRequiredUnsignedInt16Arr(uint16_t *x, int32_t length) const { if (swap_) { - for (int32_t i = 0; i < length; i++) - Swap16(reinterpret_cast(&x[i])); + for (int32_t i = 0; i < length; i++) + Swap16(reinterpret_cast(&x[i])); } } @@ -86,8 +86,8 @@ class ByteSwapper { inline void SwapIfRequiredFloatArr(float *x, int32_t length) const { if (swap_) { - for (int32_t i = 0; i < length; i++) - Swap32(reinterpret_cast(&x[i])); + for (int32_t i = 0; i < length; i++) + Swap32(reinterpret_cast(&x[i])); } } @@ -99,8 +99,8 @@ class ByteSwapper { inline void SwapIfRequiredDoubleArr(double *x, int32_t length) const { if (swap_) { - for (int32_t i = 0; i < length; i++) - Swap64(reinterpret_cast(&x[i])); + for (int32_t i = 0; i < length; i++) + Swap64(reinterpret_cast(&x[i])); } } diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc index eeb29ef30b..e48fce4ed2 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -13,9 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/ignite/kernels/ignite_dataset.h" #include #include "tensorflow/contrib/ignite/kernels/ignite_binary_object_parser.h" +#include "tensorflow/contrib/ignite/kernels/ignite_dataset.h" #include "tensorflow/core/framework/dataset.h" namespace tensorflow { diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc index 17f2bf45d1..43d6108c34 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client_windows.cc @@ -24,8 +24,8 @@ limitations under the License. #pragma comment(lib, "Mswsock.lib") #pragma comment(lib, "AdvApi32.lib") -#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/platform/logging.h" namespace tensorflow { -- GitLab From c513c04aed8790c78c46b78f90ec848555498ce4 Mon Sep 17 00:00:00 2001 From: dmitrievanthony Date: Thu, 13 Sep 2018 15:13:54 +0000 Subject: [PATCH 0056/1085] Add -DWIN32_LEAN_AND_MEAN option into BUILD. --- tensorflow/contrib/ignite/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/contrib/ignite/BUILD b/tensorflow/contrib/ignite/BUILD index 2f598b4aed..1adc6c6ccc 100644 --- a/tensorflow/contrib/ignite/BUILD +++ b/tensorflow/contrib/ignite/BUILD @@ -61,6 +61,9 @@ cc_library( "@boringssl//:ssl", "@protobuf_archive//:protobuf_headers", ], + copts = if_windows([ + "-DWIN32_LEAN_AND_MEAN", + ]), alwayslink = 1, ) -- GitLab From f54856b1448bed24534189e4aa2ebb9d0b4f5b9a Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Thu, 13 Sep 2018 18:13:47 +0000 Subject: [PATCH 0057/1085] Apply buildifier changes. --- tensorflow/contrib/ignite/BUILD | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/ignite/BUILD b/tensorflow/contrib/ignite/BUILD index 1adc6c6ccc..9393b702d1 100644 --- a/tensorflow/contrib/ignite/BUILD +++ b/tensorflow/contrib/ignite/BUILD @@ -6,14 +6,14 @@ exports_files(["LICENSE"]) load( "//tensorflow:tensorflow.bzl", - "tf_gen_op_wrapper_py", - "tf_kernel_library", + "if_not_windows", + "if_windows", "tf_custom_op_library", "tf_custom_op_py_library", "tf_gen_op_libs", + "tf_gen_op_wrapper_py", + "tf_kernel_library", "tf_py_test", - "if_not_windows", - "if_windows", ) py_library( @@ -55,15 +55,15 @@ cc_library( ]) + if_windows([ "kernels/ignite_plain_client_windows.cc", ]), + copts = if_windows([ + "-DWIN32_LEAN_AND_MEAN", + ]), deps = [ "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", "@boringssl//:ssl", "@protobuf_archive//:protobuf_headers", ], - copts = if_windows([ - "-DWIN32_LEAN_AND_MEAN", - ]), alwayslink = 1, ) -- GitLab From 1557c36e6552138ba3aacb8a56fcd082c76ed606 Mon Sep 17 00:00:00 2001 From: Rasmi Elasmar Date: Thu, 13 Sep 2018 16:45:31 -0400 Subject: [PATCH 0058/1085] Updated docs to point to tfp instead of tf.contrib --- tensorflow/contrib/bayesflow/__init__.py | 2 ++ tensorflow/contrib/bayesflow/python/ops/monte_carlo.py | 5 ++++- tensorflow/contrib/distributions/__init__.py | 2 ++ .../contrib/distributions/python/ops/bijectors/__init__.py | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/bayesflow/__init__.py b/tensorflow/contrib/bayesflow/__init__.py index 41a8c920fc..493046b399 100644 --- a/tensorflow/contrib/bayesflow/__init__.py +++ b/tensorflow/contrib/bayesflow/__init__.py @@ -14,6 +14,8 @@ # ============================================================================== """Ops for representing Bayesian computation. +Use [tfp](/probability/api_docs/python/tfp) instead. + ## This package provides classes for Bayesian computation with TensorFlow. """ from __future__ import absolute_import diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py index 68fa415eea..28a829d87d 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Monte Carlo integration and helpers.""" +"""Monte Carlo integration and helpers. + +Use [tfp.monte_carlo](/probability/api_docs/python/tfp/monte_carlo) instead. +""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py index 5cec93c4df..92bb058e17 100644 --- a/tensorflow/contrib/distributions/__init__.py +++ b/tensorflow/contrib/distributions/__init__.py @@ -13,6 +13,8 @@ # limitations under the License. # ============================================================================== """Classes representing statistical distributions and ops for working with them. + +Use [tfp.distributions](/probability/api_docs/python/tfp/distributions) instead. """ from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index e141f8b5c6..3b17de9b8a 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -14,6 +14,8 @@ # ============================================================================== """Bijector Ops. +Use [tfp.bijectors](/probability/api_docs/python/tfp/bijectors) instead. + @@AbsoluteValue @@Affine @@AffineLinearOperator -- GitLab From 8b13622dec3caaa26fec0b48079198721da5c1e7 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Sat, 15 Sep 2018 21:02:47 -0700 Subject: [PATCH 0059/1085] Raise warning once only if trainable_weights and _collected_trainable_weights are inconsistent --- tensorflow/python/keras/engine/training.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index dc464c02b6..ed51b2069e 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -676,11 +676,10 @@ class Model(Network): return if len(self.trainable_weights) != len(self._collected_trainable_weights): - logging.warning( - UserWarning( - 'Discrepancy between trainable weights and collected trainable' - ' weights, did you set `model.trainable` without calling' - ' `model.compile` after ?')) + logging.log_first_n(logging.WARN, + 'Discrepancy between trainable weights and collected trainable' + ' weights, did you set `model.trainable` without calling' + ' `model.compile` after ?', 1) def _make_train_function(self): if not hasattr(self, 'train_function'): -- GitLab From 41adb4d6c787f23f4555a692e54f62b0101a7a6f Mon Sep 17 00:00:00 2001 From: leondgarse Date: Sun, 16 Sep 2018 16:25:42 +0800 Subject: [PATCH 0060/1085] Replace multinomial in comment by argmax Replace 'a multinomial distribution' in comment by 'argmax', and also in the text introduction. --- .../python/examples/generative_examples/text_generation.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb index 07dbfd3630..ad481175fe 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb @@ -567,7 +567,7 @@ "\n", "* We get predictions using the start_string and the hidden state\n", "\n", - "* Then we use a multinomial distribution to calculate the index of the predicted word. **We use this predicted word as our next input to the model**\n", + "* Then we use argmax to calculate the index of the predicted word. **We use this predicted word as our next input to the model**\n", "\n", "* **The hidden state returned by the model is fed back into the model so that it now has more context rather than just one word.** After we predict the next word, the modified hidden states are again fed back into the model, which is how it learns as it gets more context from the previously predicted words.\n", "\n", @@ -603,7 +603,7 @@ "for i in range(num_generate):\n", " predictions, hidden = model(input_eval, hidden)\n", "\n", - " # using a multinomial distribution to predict the word returned by the model\n", + " # using argmax to predict the word returned by the model\n", " predicted_id = tf.argmax(predictions[-1]).numpy()\n", " \n", " # We pass the predicted word as the next input to the model\n", -- GitLab From 14b5eb7d0295060204bf56e041da4c84c44d8cd5 Mon Sep 17 00:00:00 2001 From: leondgarse Date: Sun, 16 Sep 2018 16:34:32 +0800 Subject: [PATCH 0061/1085] Remove a temperature line Remove a temperature line in Next Steps part --- .../python/examples/generative_examples/text_generation.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb index ad481175fe..bda9e77085 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb @@ -626,7 +626,6 @@ "\n", "* Change the start string to a different character, or the start of a sentence.\n", "* Experiment with training on a different, or with different parameters. [Project Gutenberg](http://www.gutenberg.org/ebooks/100), for example, contains a large collection of books.\n", - "* Experiment with the temperature parameter.\n", "* Add another RNN layer.\n" ] }, -- GitLab From 6ce5f3e461aad6cc8c53c661d1679d651502f4d3 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Sun, 16 Sep 2018 18:42:53 -0700 Subject: [PATCH 0062/1085] Fix line too long --- tensorflow/python/keras/engine/training.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index ed51b2069e..be03c096bf 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -677,9 +677,9 @@ class Model(Network): if len(self.trainable_weights) != len(self._collected_trainable_weights): logging.log_first_n(logging.WARN, - 'Discrepancy between trainable weights and collected trainable' - ' weights, did you set `model.trainable` without calling' - ' `model.compile` after ?', 1) + 'Discrepancy between trainable weights and collected' + ' trainable weights, did you set `model.trainable`' + ' without calling `model.compile` after ?', 1) def _make_train_function(self): if not hasattr(self, 'train_function'): -- GitLab From fa80a920f2a3bc00522fe95fc9a07a28d67fc055 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Mon, 17 Sep 2018 12:50:18 +0300 Subject: [PATCH 0063/1085] Add 'override' specifier to ReadData, WriteData. --- tensorflow/contrib/ignite/kernels/ignite_plain_client.h | 4 ++-- tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h index 750ebe605a..d12d56fdc1 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_plain_client.h +++ b/tensorflow/contrib/ignite/kernels/ignite_plain_client.h @@ -29,8 +29,8 @@ class PlainClient : public Client { virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual Status ReadData(uint8_t* buf, const int32_t length); - virtual Status WriteData(const uint8_t* buf, const int32_t length); + virtual Status ReadData(uint8_t* buf, const int32_t length) override; + virtual Status WriteData(const uint8_t* buf, const int32_t length) override; private: const string host_; diff --git a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h index d59ce91aba..372156a757 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_ssl_wrapper.h @@ -32,8 +32,8 @@ class SslWrapper : public Client { virtual Status Disconnect(); virtual bool IsConnected(); virtual int GetSocketDescriptor(); - virtual Status ReadData(uint8_t* buf, const int32_t length); - virtual Status WriteData(const uint8_t* buf, const int32_t length); + virtual Status ReadData(uint8_t* buf, const int32_t length) override; + virtual Status WriteData(const uint8_t* buf, const int32_t length) override; private: Status InitSslContext(); -- GitLab From 50762768ef9d7915ade5cf485d26ffb96753df71 Mon Sep 17 00:00:00 2001 From: Yifei Feng <1192265+yifeif@users.noreply.github.com> Date: Tue, 18 Sep 2018 00:18:57 -0700 Subject: [PATCH 0064/1085] Update CONTRIBUTING.md to reflect PR merge process --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f598999f35..3f62e3f645 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,8 @@ Follow either of the two links above to access the appropriate CLA and instructi If you have improvements to TensorFlow, send us your pull requests! For those just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). -TensorFlow team members will be assigned to review your pull requests. Once the pull requests are approved and pass continuous integration checks, we will merge the pull requests. +TensorFlow team members will be assigned to review your pull requests. Once the pull requests are approved and pass continuous integration checks, a TensorFlow team member will apply `ready to pull` to your change. This means we are working on getting your pull request submitted to our internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. + For some pull requests, we will apply the patch for each pull request to our internal version control system first, and export the change out as a new commit later, at which point the original pull request will be closed. The commits in the pull request will be squashed into a single commit with the pull request creator as the author. These pull requests will be labeled as pending merge internally. If you want to contribute but you're not sure where to start, take a look at the -- GitLab From a8fe42cdfc7341655f61414ce02ddd9d016165ed Mon Sep 17 00:00:00 2001 From: Yifei Feng <1192265+yifeif@users.noreply.github.com> Date: Tue, 18 Sep 2018 00:19:38 -0700 Subject: [PATCH 0065/1085] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f62e3f645..05e970e8cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ Follow either of the two links above to access the appropriate CLA and instructi If you have improvements to TensorFlow, send us your pull requests! For those just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). -TensorFlow team members will be assigned to review your pull requests. Once the pull requests are approved and pass continuous integration checks, a TensorFlow team member will apply `ready to pull` to your change. This means we are working on getting your pull request submitted to our internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. +TensorFlow team members will be assigned to review your pull requests. Once the pull requests are approved and pass continuous integration checks, a TensorFlow team member will apply `ready to pull` label to your change. This means we are working on getting your pull request submitted to our internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. For some pull requests, we will apply the patch for each pull request to our internal version control system first, and export the change out as a new commit later, at which point the original pull request will be closed. The commits in the pull request will be squashed into a single commit with the pull request creator as the author. These pull requests will be labeled as pending merge internally. -- GitLab From d28a29e2c3fbc19ab6207e9075e2c3ccfdc415ed Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Tue, 18 Sep 2018 10:16:14 +0200 Subject: [PATCH 0066/1085] indentation fix --- tensorflow/python/keras/optimizers.py | 76 +++++++++++++-------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py index 629972fd8e..68081e2485 100644 --- a/tensorflow/python/keras/optimizers.py +++ b/tensorflow/python/keras/optimizers.py @@ -314,24 +314,24 @@ class RMSprop(Optimizer): @tf_export('keras.optimizers.Adagrad') class Adagrad(Optimizer): - """Adagrad optimizer. + """Adagrad optimizer. - Adagrad is an optimizer with parameter-specific learning rates, - which are adapted relative to how frequently a parameter gets - updated during training. The more updates a parameter receives, - the smaller the updates. + Adagrad is an optimizer with parameter-specific learning rates, + which are adapted relative to how frequently a parameter gets + updated during training. The more updates a parameter receives, + the smaller the updates. - It is recommended to leave the parameters of this optimizer - at their default values. + It is recommended to leave the parameters of this optimizer + at their default values. - # Arguments - lr: float >= 0. Initial learning rate. - epsilon: float >= 0. If `None`, defaults to `K.epsilon()`. - decay: float >= 0. Learning rate decay over each update. + # Arguments + lr: float >= 0. Initial learning rate. + epsilon: float >= 0. If `None`, defaults to `K.epsilon()`. + decay: float >= 0. Learning rate decay over each update. - # References - - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) - """ + # References + - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + """ def __init__(self, lr=0.01, epsilon=None, decay=0., **kwargs): super(Adagrad, self).__init__(**kwargs) @@ -381,30 +381,30 @@ class Adagrad(Optimizer): @tf_export('keras.optimizers.Adadelta') class Adadelta(Optimizer): - """Adadelta optimizer. - - Adadelta is a more robust extension of Adagrad - that adapts learning rates based on a moving window of gradient updates, - instead of accumulating all past gradients. This way, Adadelta continues - learning even when many updates have been done. Compared to Adagrad, in the - original version of Adadelta you don't have to set an initial learning - rate. In this version, initial learning rate and decay factor can - be set, as in most other Keras optimizers. - - It is recommended to leave the parameters of this optimizer - at their default values. - - # Arguments - lr: float >= 0. Initial learning rate, defaults to 1. - It is recommended to leave it at the default value. - rho: float >= 0. Adadelta decay factor, corresponding to fraction of - gradient to keep at each time step. - epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. - decay: float >= 0. Initial learning rate decay. - - # References - - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701) - """ + """Adadelta optimizer. + + Adadelta is a more robust extension of Adagrad + that adapts learning rates based on a moving window of gradient updates, + instead of accumulating all past gradients. This way, Adadelta continues + learning even when many updates have been done. Compared to Adagrad, in the + original version of Adadelta you don't have to set an initial learning + rate. In this version, initial learning rate and decay factor can + be set, as in most other Keras optimizers. + + It is recommended to leave the parameters of this optimizer + at their default values. + + # Arguments + lr: float >= 0. Initial learning rate, defaults to 1. + It is recommended to leave it at the default value. + rho: float >= 0. Adadelta decay factor, corresponding to fraction of + gradient to keep at each time step. + epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. + decay: float >= 0. Initial learning rate decay. + + # References + - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701) + """ def __init__(self, lr=1.0, rho=0.95, epsilon=None, decay=0., **kwargs): super(Adadelta, self).__init__(**kwargs) -- GitLab From 6d67ba41f566e963e2c061ca7df63edad89e1fca Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Tue, 18 Sep 2018 18:56:55 +0300 Subject: [PATCH 0067/1085] Work out the endianness statically. --- tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h index 5b42de4c5a..484cc4d6f5 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h @@ -20,12 +20,12 @@ limitations under the License. namespace tensorflow { +constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; + class ByteSwapper { public: ByteSwapper(bool big_endian) { - int x = 1; - bool is_little_endian = (*(char *)&x == 1); - swap_ = big_endian == is_little_endian; + swap_ = big_endian == kLittleEndian; } inline void SwapIfRequiredInt16(int16_t *x) const { -- GitLab From 14e9345a88b08f5d2a12f3f441b1d82c041d7ea3 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Tue, 18 Sep 2018 18:23:52 +0000 Subject: [PATCH 0068/1085] Avoid saving sensitive information in graph. --- .../ignite/kernels/ignite_dataset_ops.cc | 30 ++------- tensorflow/contrib/ignite/ops/dataset_ops.cc | 10 --- .../ignite/python/ops/ignite_dataset_ops.py | 18 +---- .../python/tests/ignite_dataset_test.py | 66 ++++++++++++++----- 4 files changed, 56 insertions(+), 68 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc index e48fce4ed2..bdaed72387 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc +++ b/tensorflow/contrib/ignite/kernels/ignite_dataset_ops.cc @@ -125,35 +125,15 @@ class IgniteDatasetOp : public DatasetOpKernel { OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "page_size", &page_size)); - if (env_username) - username = string(env_username); - else - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "username", &username)); + if (env_username) username = string(env_username); - if (env_password) - password = string(env_password); - else - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "password", &password)); + if (env_password) password = string(env_password); - if (env_certfile) - certfile = string(env_certfile); - else - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "certfile", &certfile)); + if (env_certfile) certfile = string(env_certfile); - if (env_keyfile) - keyfile = string(env_keyfile); - else - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "keyfile", &keyfile)); + if (env_keyfile) keyfile = string(env_keyfile); - if (env_cert_password) - cert_password = string(env_cert_password); - else - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "cert_password", - &cert_password)); + if (env_cert_password) cert_password = string(env_cert_password); const Tensor* schema_tensor; OP_REQUIRES_OK(ctx, ctx->input("schema", &schema_tensor)); diff --git a/tensorflow/contrib/ignite/ops/dataset_ops.cc b/tensorflow/contrib/ignite/ops/dataset_ops.cc index 7d18df11aa..3d6fbe00e6 100644 --- a/tensorflow/contrib/ignite/ops/dataset_ops.cc +++ b/tensorflow/contrib/ignite/ops/dataset_ops.cc @@ -26,11 +26,6 @@ REGISTER_OP("IgniteDataset") .Input("local: bool") .Input("part: int32") .Input("page_size: int32") - .Input("username: string") - .Input("password: string") - .Input("certfile: string") - .Input("keyfile: string") - .Input("cert_password: string") .Input("schema: int32") .Input("permutation: int32") .Output("handle: variant") @@ -54,11 +49,6 @@ port: Ignite Thin Client Port. local: Local flag that defines that data should be fetched from local host only. part: Partition data should be fetched from. page_size: Page size for Ignite Thin Client. -username: Username to authenticate via Ignite Thin Client. -password: Password to authenticate via Ignite Thin Client. -certfile: SSL certificate to establish SSL connection. -keyfile: Private key file to establish SSL connection. -cert_password: SSL certificate password to establish SSL connection. schema: Internal structure that defines schema of cache objects. permutation: Internal structure that defines permutation of cache objects. )doc"); diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py index c0e24b1c69..7fc9e1fdd1 100644 --- a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py +++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py @@ -732,18 +732,6 @@ class IgniteDataset(Dataset): self.part = ops.convert_to_tensor(part, dtype=dtypes.int32, name="part") self.page_size = ops.convert_to_tensor(page_size, dtype=dtypes.int32, name="page_size") - self.username = ops.convert_to_tensor("" if username is None else username, - dtype=dtypes.string, name="username") - self.password = ops.convert_to_tensor("" if password is None else password, - dtype=dtypes.string, name="password") - self.certfile = ops.convert_to_tensor("" if certfile is None else certfile, - dtype=dtypes.string, name="certfile") - self.keyfile = ops.convert_to_tensor("" if keyfile is None else keyfile, - dtype=dtypes.string, name="keyfile") - self.cert_password = ops.convert_to_tensor("" if cert_password is None - else cert_password, - dtype=dtypes.string, - name="cert_password") self.schema = ops.convert_to_tensor(self.cache_type.to_flat(), dtype=dtypes.int32, name="schema") self.permutation = ops.convert_to_tensor(self.cache_type.to_permutation(), @@ -753,10 +741,8 @@ class IgniteDataset(Dataset): def _as_variant_tensor(self): return gen_dataset_ops.ignite_dataset(self.cache_name, self.host, self.port, self.local, self.part, - self.page_size, self.username, - self.password, self.certfile, - self.keyfile, self.cert_password, - self.schema, self.permutation) + self.page_size, self.schema, + self.permutation) @property def output_classes(self): diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py index 933e62b804..5d74617690 100644 --- a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py +++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py @@ -35,28 +35,60 @@ class IgniteDatasetTest(test.TestCase): """ def test_ignite_dataset_with_plain_client(self): + """Test Ignite Dataset with plain client. + """ + self._clear_env() ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42300) - self.__check_dataset(ds) + self._check_dataset(ds) def test_ignite_dataset_with_ssl_client(self): - ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42301,\ - certfile=os.path.dirname(os.path.realpath(__file__)) +\ - "/keystore/client.pem", cert_password="123456") - self.__check_dataset(ds) + """Test Ignite Dataset with ssl client. + """ + self._clear_env() + os.environ["IGNITE_DATASET_CERTFILE"] = os.path.dirname( + os.path.realpath(__file__)) + "/keystore/client.pem" + os.environ["IGNITE_DATASET_CERT_PASSWORD"] = "123456" + + ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42301, + certfile=os.environ["IGNITE_DATASET_CERTFILE"], + cert_password=os.environ["IGNITE_DATASET_CERT_PASSWORD"]) + self._check_dataset(ds) def test_ignite_dataset_with_ssl_client_and_auth(self): - ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42302,\ - certfile=os.path.dirname(os.path.realpath(__file__)) +\ - "/keystore/client.pem", cert_password="123456",\ - username="ignite", password="ignite") - self.__check_dataset(ds) + """Test Ignite Dataset with ssl client and authentication. + """ + self._clear_env() + os.environ['IGNITE_DATASET_USERNAME'] = "ignite" + os.environ['IGNITE_DATASET_PASSWORD'] = "ignite" + os.environ['IGNITE_DATASET_CERTFILE'] = os.path.dirname( + os.path.realpath(__file__)) + "/keystore/client.pem" + os.environ['IGNITE_DATASET_CERT_PASSWORD'] = "123456" + + ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42302, + certfile=os.environ['IGNITE_DATASET_CERTFILE'], + cert_password=os.environ['IGNITE_DATASET_CERT_PASSWORD'], + username=os.environ['IGNITE_DATASET_USERNAME'], + password=os.environ['IGNITE_DATASET_PASSWORD']) + self._check_dataset(ds) + + def _clear_env(self): + """Clears environment variables used by Ignite Dataset. + """ + if 'IGNITE_DATASET_USERNAME' in os.environ: + del os.environ['IGNITE_DATASET_USERNAME'] + if 'IGNITE_DATASET_PASSWORD' in os.environ: + del os.environ['IGNITE_DATASET_PASSWORD'] + if 'IGNITE_DATASET_CERTFILE' in os.environ: + del os.environ['IGNITE_DATASET_CERTFILE'] + if 'IGNITE_DATASET_CERT_PASSWORD' in os.environ: + del os.environ['IGNITE_DATASET_CERT_PASSWORD'] - def __check_dataset(self, dataset): + def _check_dataset(self, dataset): """Checks that dataset provids correct data. """ - self.assertEquals(tf.int64, dataset.output_types['key']) - self.assertEquals(tf.string, dataset.output_types['val']['NAME']) - self.assertEquals(tf.int64, dataset.output_types['val']['VAL']) + self.assertEqual(tf.int64, dataset.output_types['key']) + self.assertEqual(tf.string, dataset.output_types['val']['NAME']) + self.assertEqual(tf.int64, dataset.output_types['val']['VAL']) it = dataset.make_one_shot_iterator() ne = it.get_next() @@ -66,11 +98,11 @@ class IgniteDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(ne) - self.assertEquals({'key': 1, 'val': {'NAME': b'TEST1', 'VAL': 42}},\ + self.assertEqual({'key': 1, 'val': {'NAME': b'TEST1', 'VAL': 42}},\ rows[0]) - self.assertEquals({'key': 2, 'val': {'NAME': b'TEST2', 'VAL': 43}},\ + self.assertEqual({'key': 2, 'val': {'NAME': b'TEST2', 'VAL': 43}},\ rows[1]) - self.assertEquals({'key': 3, 'val': {'NAME': b'TEST3', 'VAL': 44}},\ + self.assertEqual({'key': 3, 'val': {'NAME': b'TEST3', 'VAL': 44}},\ rows[2]) if __name__ == "__main__": -- GitLab From 1e821cd9a02b59a90a8b983759cf74eded16265f Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Wed, 19 Sep 2018 11:06:40 -0700 Subject: [PATCH 0069/1085] Fix bug in metrics sparse_categorical_accuracy and sparse_top_k_categorical_accuracy --- tensorflow/python/keras/metrics.py | 15 ++++++++------ tensorflow/python/keras/metrics_test.py | 26 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py index e64241e5cf..2fd3244800 100644 --- a/tensorflow/python/keras/metrics.py +++ b/tensorflow/python/keras/metrics.py @@ -635,7 +635,9 @@ def categorical_accuracy(y_true, y_pred): @tf_export('keras.metrics.sparse_categorical_accuracy') def sparse_categorical_accuracy(y_true, y_pred): - y_true = math_ops.reduce_max(y_true, axis=-1) + # If the shape of y_true is (num_samples, 1), squeeze to (num_samples,) + if (len(K.int_shape(y_true)) == len(K.int_shape(y_pred))): + y_true = array_ops.squeeze(y_true, [-1]) y_pred = math_ops.argmax(y_pred, axis=-1) # If the expected labels are float, we need to cast the int returned by @@ -654,11 +656,12 @@ def top_k_categorical_accuracy(y_true, y_pred, k=5): @tf_export('keras.metrics.sparse_top_k_categorical_accuracy') def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5): - return K.mean( - nn.in_top_k(y_pred, - math_ops.cast(math_ops.reduce_max(y_true, axis=-1), 'int32'), - k), - axis=-1) + # If the shape of y_true is (num_samples, 1), squeeze to (num_samples,) + if (len(K.int_shape(y_true)) == len(K.int_shape(y_pred))): + y_true = array_ops.squeeze(y_true, [-1]) + + return K.mean(nn.in_top_k(y_pred, math_ops.cast(y_true, 'int32'), k), + axis=-1) # Aliases diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py index 4195ea18ad..43ac5b7ead 100644 --- a/tensorflow/python/keras/metrics_test.py +++ b/tensorflow/python/keras/metrics_test.py @@ -54,6 +54,18 @@ class KerasMetricsTest(test.TestCase): y_pred = K.variable(np.random.random((6, 7))) self.assertEqual(K.eval(metric(y_true, y_pred)).shape, (6,)) + # Test correctness if the shape of y_true is (num_samples,) + y_true = K.variable([1., 0., 0., 0.]) + y_pred = K.variable([[0.8, 0.2], [0.6, 0.4], [0.7, 0.3], [0.9, 0.1]]) + print(K.eval(metric(y_true, y_pred))) + self.assertAllEqual(K.eval(metric(y_true, y_pred)), [0., 1., 1., 1.]) + + # Test correctness if the shape of y_true is (num_samples, 1) + y_true = K.variable([[1.], [0.], [0.], [0.]]) + y_pred = K.variable([[0.8, 0.2], [0.6, 0.4], [0.7, 0.3], [0.9, 0.1]]) + print(K.eval(metric(y_true, y_pred))) + self.assertAllEqual(K.eval(metric(y_true, y_pred)), [0., 1., 1., 1.]) + def test_sparse_categorical_accuracy_float(self): with self.cached_session(): metric = metrics.sparse_categorical_accuracy @@ -79,6 +91,7 @@ class KerasMetricsTest(test.TestCase): def test_sparse_top_k_categorical_accuracy(self): with self.cached_session(): + # Test correctness if the shape of y_true is (num_samples, 1) y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([[1], [0]])) result = K.eval( @@ -91,6 +104,19 @@ class KerasMetricsTest(test.TestCase): metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1)) self.assertEqual(result, 0.) + # Test correctness if the shape of y_true is (num_samples,) + y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) + y_true = K.variable(np.array([1, 0])) + result = K.eval( + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3)) + self.assertEqual(result, 1) + result = K.eval( + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2)) + self.assertEqual(result, 0.5) + result = K.eval( + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1)) + self.assertEqual(result, 0.) + def test_top_k_categorical_accuracy(self): with self.cached_session(): y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) -- GitLab From 78e205d35b31aa49e8dac357d827900a165f0a21 Mon Sep 17 00:00:00 2001 From: Erik Smistad Date: Thu, 20 Sep 2018 15:56:34 +0200 Subject: [PATCH 0070/1085] Added warning message if cmake version is below 3.8 or host toolset is not set to x64 on windows --- tensorflow/contrib/cmake/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 225c5e6227..a7a66472df 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -1,8 +1,14 @@ # Minimum CMake required +cmake_minimum_required(VERSION 3.5) + if(WIN32) - cmake_minimum_required(VERSION 3.8) -else() - cmake_minimum_required(VERSION 3.5) + if(${CMAKE_VERSION} VERSION_LESS "3.8") + message(WARNING "Your current cmake version is ${CMAKE_VERSION} which does not support setting the toolset architecture to x64. This may cause \"compiler out of heap space\" errors when building. Consider upgrading your cmake to > 3.8 and using the flag -Thost=x64 when running cmake.") + else() + if(NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE OR NOT "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE}" STREQUAL "x64") + message(WARNING "Your current cmake generator is set to use 32 bit toolset architecture. This may cause \"compiler out of heap space\" errors when building. Consider using the flag -Thost=x64 when running cmake.") + endif() + endif() endif() # Project -- GitLab From 039ddaa6c0af4be4291383564db5a964d0035c1d Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Thu, 20 Sep 2018 15:49:40 -0700 Subject: [PATCH 0071/1085] Fix bad indentation --- tensorflow/python/keras/metrics_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py index 43ac5b7ead..5f5565d4d5 100644 --- a/tensorflow/python/keras/metrics_test.py +++ b/tensorflow/python/keras/metrics_test.py @@ -108,13 +108,13 @@ class KerasMetricsTest(test.TestCase): y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([1, 0])) result = K.eval( - metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3)) + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3)) self.assertEqual(result, 1) result = K.eval( - metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2)) + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2)) self.assertEqual(result, 0.5) result = K.eval( - metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1)) + metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1)) self.assertEqual(result, 0.) def test_top_k_categorical_accuracy(self): -- GitLab From 6ee2153dea6e094fd1a9667ac3bfabbce368a70d Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Fri, 20 Jul 2018 14:38:00 -0700 Subject: [PATCH 0072/1085] Numpy ndarray should be serialized as Python list --- tensorflow/python/keras/engine/saving_test.py | 22 +++++++++++++++++++ tensorflow/python/util/serialization.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py index 02d99d5d69..6ff54a94ca 100644 --- a/tensorflow/python/keras/engine/saving_test.py +++ b/tensorflow/python/keras/engine/saving_test.py @@ -679,6 +679,28 @@ class TestWholeModelSaving(test.TestCase): os.remove(fname) + def test_saving_constant_initializer_with_numpy(self): + if h5py is None: + self.skipTest('h5py required to run this test') + + with self.test_session(): + model = keras.models.Sequential() + model.add( + keras.layers.Dense( + 2, + input_shape=(3,), + kernel_initializer=keras.initializers.Constant(np.ones((3, 2))) + ) + ) + model.add(keras.layers.Dense(3)) + model.compile(loss='mse', optimizer='sgd', metrics=['acc']) + fd, fname = tempfile.mkstemp('.h5') + keras.models.save_model(model, fname) + model = keras.models.load_model(fname) + os.close(fd) + os.remove(fname) + + class SubclassedModel(training.Model): def __init__(self): diff --git a/tensorflow/python/util/serialization.py b/tensorflow/python/util/serialization.py index faf5164faa..cff864c030 100644 --- a/tensorflow/python/util/serialization.py +++ b/tensorflow/python/util/serialization.py @@ -43,7 +43,7 @@ def get_json_type(obj): # if obj is any numpy type if type(obj).__module__ == np.__name__: if isinstance(obj, np.ndarray): - return {'type': type(obj), 'value': obj.tolist()} + return obj.tolist() else: return obj.item() -- GitLab From d6f93dfd0bb784a3d99105f8f5989ff08f33119e Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Wed, 15 Aug 2018 17:31:35 -0700 Subject: [PATCH 0073/1085] Fix bad indentation --- tensorflow/python/keras/engine/saving_test.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py index 6ff54a94ca..4be370cc13 100644 --- a/tensorflow/python/keras/engine/saving_test.py +++ b/tensorflow/python/keras/engine/saving_test.py @@ -680,25 +680,25 @@ class TestWholeModelSaving(test.TestCase): def test_saving_constant_initializer_with_numpy(self): - if h5py is None: - self.skipTest('h5py required to run this test') + if h5py is None: + self.skipTest('h5py required to run this test') - with self.test_session(): - model = keras.models.Sequential() - model.add( - keras.layers.Dense( - 2, - input_shape=(3,), - kernel_initializer=keras.initializers.Constant(np.ones((3, 2))) - ) - ) - model.add(keras.layers.Dense(3)) - model.compile(loss='mse', optimizer='sgd', metrics=['acc']) - fd, fname = tempfile.mkstemp('.h5') - keras.models.save_model(model, fname) - model = keras.models.load_model(fname) - os.close(fd) - os.remove(fname) + with self.test_session(): + model = keras.models.Sequential() + model.add( + keras.layers.Dense( + 2, + input_shape=(3,), + kernel_initializer=keras.initializers.Constant(np.ones((3, 2))) + ) + ) + model.add(keras.layers.Dense(3)) + model.compile(loss='mse', optimizer='sgd', metrics=['acc']) + fd, fname = tempfile.mkstemp('.h5') + keras.models.save_model(model, fname) + model = keras.models.load_model(fname) + os.close(fd) + os.remove(fname) class SubclassedModel(training.Model): -- GitLab From 510c117752a681e80e26cf8cf3614e82d59d1e53 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Fri, 21 Sep 2018 11:32:20 -0700 Subject: [PATCH 0074/1085] Change test_session to cached_session --- tensorflow/python/keras/engine/saving_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py index 4be370cc13..8d179aea87 100644 --- a/tensorflow/python/keras/engine/saving_test.py +++ b/tensorflow/python/keras/engine/saving_test.py @@ -683,7 +683,7 @@ class TestWholeModelSaving(test.TestCase): if h5py is None: self.skipTest('h5py required to run this test') - with self.test_session(): + with self.cached_session(): model = keras.models.Sequential() model.add( keras.layers.Dense( -- GitLab From 268bf6b118646c8e93162d591263bca907c7db28 Mon Sep 17 00:00:00 2001 From: AG Ramesh Date: Fri, 21 Sep 2018 11:39:29 -0700 Subject: [PATCH 0075/1085] Removing dead code. With the addition of mkl slice using MKL DNN this code will not longer be executed --- tensorflow/core/kernels/slice_op.cc | 198 ---------------------------- 1 file changed, 198 deletions(-) diff --git a/tensorflow/core/kernels/slice_op.cc b/tensorflow/core/kernels/slice_op.cc index 77594479cb..83377ffab5 100644 --- a/tensorflow/core/kernels/slice_op.cc +++ b/tensorflow/core/kernels/slice_op.cc @@ -228,190 +228,6 @@ class SliceOp : public OpKernel { } }; -#ifdef INTEL_MKL -template -class MklSliceOp : public OpKernel { - public: - explicit MklSliceOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - TensorShape output_shape; - gtl::InlinedVector begin; - gtl::InlinedVector size; - Tensor* result = nullptr; - bool done = false; - SharedSliceCommonCases(context, &output_shape, &begin, &size, &result, - &done); - if (!context->status().ok() || done == true) return; - - const Tensor& input = context->input(0); - const int input_dims = input.dims(); - - if (output_shape.num_elements() > 0) { - if (std::is_same::value && input_dims == 2 && - DataTypeCanUseMemcpy(DataTypeToEnum::v())) { - auto input = context->input(0).tensor(); - auto output = result->tensor(); - // TODO(agarwal): Consider multi-threading this loop for cases where - // size[0] is very large. - for (int i = 0; i < size[0]; ++i) { - const int64 row = begin[0] + i; - if (i + 1 < size[0]) { - port::prefetch(&output(i + 1, 0)); - port::prefetch(&input(row + 1, begin[1])); - } - memcpy(&output(i, 0), &input(row, begin[1]), size[1] * sizeof(T)); - } - return; - } -#define HANDLE_DIM(NDIM) \ - if (input_dims == NDIM) { \ - HandleCase(context, begin, size, result); \ - return; \ - } - - HANDLE_DIM(1); - HANDLE_DIM(2); - HANDLE_DIM(3); - HANDLE_DIM(4); - HANDLE_DIM(5); - HANDLE_DIM(6); - HANDLE_DIM(7); - -#undef HANDLE_DIM - - OP_REQUIRES( - context, false, - errors::Unimplemented("SliceOp : Unhandled input dimensions")); - } - } - - private: - // Helper function for DoesSliceShapeDifferInOnly1D. Checks if the following - // criteria matches for slice_dim: if indices for slice are 0 in all dims - // except slice_dim and if sizes of all the dimensions of the slice are same - // as the sizes of all the dimensions of the input except slice_dim, then - // returns True. Otherwise, returns False. - bool DoesSliceShapeDifferInOnly1DHelper(const TensorShape& input_shape, - const gtl::ArraySlice& begin, - const gtl::ArraySlice& size, - int slice_dim) { - for (int dim = 0; dim < 4; dim++) { - if (dim != slice_dim && - (begin[dim] != 0 || size[dim] != input_shape.dim_size(dim))) { - return false; - } - } - return true; - } - - // Is 'input' tensor being sliced over a single dimension out of 4? - // - // This check is applicable in the context of Slice of a 4-D tensor in - // NHWC or NCHW format over channel dimension. - // - // If indices for slice are 0 in all dims except one dimension and if sizes of - // all dimensions of slice are same as sizes of all dimensions of inputs - // except that dimension, then we are slicing over a single dimension. - // - // Returns True if Slicing over a single dimension, and sets slice_dim - // to the number of the dimension that satisfies criteria. - bool DoesSliceShapeDifferInOnly1D(const TensorShape& input_shape, - const gtl::ArraySlice& begin, - const gtl::ArraySlice& size, - int* slice_dim) { - for (int dim = 0; dim < 4; dim++) { - if (DoesSliceShapeDifferInOnly1DHelper(input_shape, begin, size, dim)) { - *slice_dim = dim; - return true; - } - } - return false; - } - - template - void HandleCase(OpKernelContext* context, const gtl::ArraySlice& begin, - const gtl::ArraySlice& size, Tensor* result) { - int slice_dim = -1; - TensorShape in_shape = context->input(0).shape(); - // Special case for handling 4-D tensor slice when shape of the slice - // differs from the input tensor in only 1 out of 4 dimensions. - // This case arises in the context of Slice of 4-D tensor in NHWC or NCHW - // format over channel dimension. - if (NDIM == 4 && - DoesSliceShapeDifferInOnly1D(in_shape, begin, size, &slice_dim)) { - size_t in_strides[4] = { - (size_t)in_shape.dim_size(1) * in_shape.dim_size(2) * - in_shape.dim_size(3), - (size_t)in_shape.dim_size(2) * in_shape.dim_size(3), - (size_t)in_shape.dim_size(3), (size_t)1}; - - size_t out_strides[4] = {(size_t)size[1] * size[2] * size[3], - (size_t)size[2] * size[3], (size_t)size[3], - (size_t)1}; - - T* in_buf = const_cast( - const_cast(context->input(0).flat().data())); - T* op_buf = result->flat().data(); - - if (slice_dim == 1) { - /* data format = NCHW */ - -#pragma omp parallel for - for (ssize_t d0 = begin[0]; d0 < begin[0] + size[0]; d0++) { - T* ip = in_buf + (d0 * in_strides[0]); - T* op = op_buf + ((d0 - begin[0]) * out_strides[0]); -#pragma omp parallel for - for (ssize_t d1 = begin[1]; d1 < begin[1] + size[1]; d1++) { - T* ip1 = ip + (d1 * in_strides[1]); - T* op1 = op + ((d1 - begin[1]) * out_strides[1]); - // For NCHW, H and W will be contiguous. So we can copy - // both with one memcpy. - memcpy(static_cast(op1), static_cast(ip1), - sizeof(T) * in_strides[1]); - } - } - return; - } else if (slice_dim == 3) { - /* data_format = NHWC */ - -#pragma omp parallel for - for (ssize_t d0 = begin[0]; d0 < begin[0] + size[0]; d0++) { - T* ip = in_buf + (d0 * in_strides[0]); - T* op = op_buf + ((d0 - begin[0]) * out_strides[0]); -#pragma omp parallel for - for (ssize_t d1 = begin[1]; d1 < begin[1] + size[1]; d1++) { - T* ip1 = ip + (d1 * in_strides[1]); - T* op1 = op + ((d1 - begin[1]) * out_strides[1]); -#pragma omp parallel for - for (ssize_t d2 = begin[2]; d2 < begin[2] + size[2]; d2++) { - T* ip2 = ip1 + (d2 * in_strides[2]); - T* ip3 = ip2 + begin[3]; - T* op2 = op1 + ((d2 - begin[2]) * out_strides[2]); - T* op3 = op2; - memcpy(static_cast(op3), static_cast(ip3), - sizeof(T) * size[3]); - } - } - } - return; - } - // slice_dim is not 1 or 3, then we fallback to Eigen implementation. - } - - Eigen::DSizes indices; - Eigen::DSizes sizes; - for (int i = 0; i < NDIM; ++i) { - indices[i] = begin[i]; - sizes[i] = size[i]; - } - - functor::Slice()( - context->eigen_device(), result->tensor(), - context->input(0).tensor(), indices, sizes); - } -}; -#endif // Forward declarations of the functor specializations for declared in the // sharded source files. @@ -440,7 +256,6 @@ TF_CALL_ALL_TYPES(DECLARE_FOR_N); #undef DECLARE_CPU_SPEC } // namespace functor -#ifndef INTEL_MKL #define REGISTER_SLICE(type) \ REGISTER_KERNEL_BUILDER(Name("Slice") \ .Device(DEVICE_CPU) \ @@ -452,19 +267,6 @@ TF_CALL_ALL_TYPES(DECLARE_FOR_N); TF_CALL_POD_STRING_TYPES(REGISTER_SLICE); TF_CALL_QUANTIZED_TYPES(REGISTER_SLICE); #undef REGISTER_SLICE -#else -#define REGISTER_SLICE(type) \ - REGISTER_KERNEL_BUILDER(Name("Slice") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .HostMemory("begin") \ - .HostMemory("size"), \ - MklSliceOp) - -TF_CALL_POD_STRING_TYPES(REGISTER_SLICE); -TF_CALL_QUANTIZED_TYPES(REGISTER_SLICE); -#undef REGISTER_SLICE -#endif // INTEL_MKL #if GOOGLE_CUDA // Forward declarations of the functor specializations for GPU. -- GitLab From 457ef66c2d4985000aa1d1a9bc643f66bbddd46d Mon Sep 17 00:00:00 2001 From: Martin Wicke <577277+martinwicke@users.noreply.github.com> Date: Fri, 21 Sep 2018 12:58:32 -0700 Subject: [PATCH 0076/1085] Fix long lines --- tensorflow/python/keras/layers/embeddings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/layers/embeddings.py b/tensorflow/python/keras/layers/embeddings.py index a0b9393812..76e551a7ce 100644 --- a/tensorflow/python/keras/layers/embeddings.py +++ b/tensorflow/python/keras/layers/embeddings.py @@ -142,12 +142,14 @@ class Embedding(Layer): else: in_lens = [self.input_length] if len(in_lens) != len(input_shape) - 1: - raise ValueError('"input_length" is %s, but received input has shape %s' % + raise ValueError('"input_length" is %s, ' + 'but received input has shape %s' % (str(self.input_length), str(input_shape))) else: for i, (s1, s2) in enumerate(zip(in_lens, input_shape[1:])): if s1 is not None and s2 is not None and s1 != s2: - raise ValueError('"input_length" is %s, but received input has shape %s' % + raise ValueError('"input_length" is %s, ' + 'but received input has shape %s' % (str(self.input_length), str(input_shape))) elif s1 is None: in_lens[i] = s2 -- GitLab From 282d6e7c384c83f9b6bf43b7b37eb606ccc64d06 Mon Sep 17 00:00:00 2001 From: Martin Wicke <577277+martinwicke@users.noreply.github.com> Date: Fri, 21 Sep 2018 12:59:15 -0700 Subject: [PATCH 0077/1085] Fix long lines --- tensorflow/python/ops/nn_ops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 17e10995f2..a68422c315 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -818,12 +818,14 @@ class Convolution(object): try: input_shape.with_rank(num_spatial_dims + 2) except ValueError: - raise ValueError("input tensor must have rank %d" % (num_spatial_dims + 2)) + raise ValueError("input tensor must have rank %d" % + (num_spatial_dims + 2)) try: filter_shape.with_rank(num_spatial_dims + 2) except ValueError: - raise ValueError("filter tensor must have rank %d" % (num_spatial_dims + 2)) + raise ValueError("filter tensor must have rank %d" % + (num_spatial_dims + 2)) if data_format is None or not data_format.startswith("NC"): input_channels_dim = input_shape[num_spatial_dims + 1] -- GitLab From 6dd7a09211cc74d11ff1554624b527c432020cbc Mon Sep 17 00:00:00 2001 From: wangsiyu Date: Sun, 23 Sep 2018 20:33:19 +0800 Subject: [PATCH 0078/1085] Enable partitioned variable assignments --- .../python/kernel_tests/variables_test.py | 43 ++++++++++++++++- tensorflow/python/ops/variables.py | 47 +++++++++++++++++-- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py index 2e7975667c..687784c8b7 100644 --- a/tensorflow/python/kernel_tests/variables_test.py +++ b/tensorflow/python/kernel_tests/variables_test.py @@ -673,7 +673,7 @@ class PartitionedVariableTest(test.TestCase): v0._set_save_slice_info( variables.Variable.SaveSliceInfo(v0.name, [2], [0], [1])) v1._set_save_slice_info( - variables.Variable.SaveSliceInfo(v0.name, [2], [1], [1])) + variables.Variable.SaveSliceInfo(v1.name, [2], [1], [1])) partitions = [2] variables.PartitionedVariable( @@ -696,6 +696,47 @@ class PartitionedVariableTest(test.TestCase): variable_list=[v0], partitions=partitions) + def testPartitionedVariableAssignments(self): + with ops.Graph().as_default(), self.cached_session() as sess: + v0 = variables.Variable(initial_value=[0.0]) + v1 = variables.Variable(initial_value=[1.0]) + v0._set_save_slice_info( + variables.Variable.SaveSliceInfo(v0.name, [2], [0], [1])) + v1._set_save_slice_info( + variables.Variable.SaveSliceInfo(v0.name, [2], [1], [1])) + partitions = [2] + + # Pass variable_list as [v1, v0] to ensure they are properly + # re-sorted to [v0, v1] based on their slice info offsets. + partitioned_variable = variables.PartitionedVariable( + name="two_vars", + shape=[2], + dtype=v0.dtype, + variable_list=[v0, v1], + partitions=partitions) + + deltas_a = constant_op.constant([1.0, 2.0]) + deltas_b = constant_op.constant([3.0, 4.0]) + ones = array_ops.ones([2]) + plus_delta = partitioned_variable.assign_add(deltas_a) + minus_delta = partitioned_variable.assign_sub(deltas_b) + assign_ones = partitioned_variable.assign(ones) + variables.global_variables_initializer().run() + + self.assertEqual([1.0], plus_delta[0].eval()) + self.assertEqual([1.0], v0.eval()) + self.assertEqual([3.0], plus_delta[1].eval()) + self.assertEqual([3.0], v1.eval()) + + self.assertEqual([-2.0], minus_delta[0].eval()) + self.assertEqual([-2.0], v0.eval()) + self.assertEqual([-1.0], minus_delta[1].eval()) + self.assertEqual([-1.0], v1.eval()) + + self.assertEqual([1.0], assign_ones[0].eval()) + self.assertEqual([1.0], v0.eval()) + self.assertEqual([1.0], assign_ones[1].eval()) + self.assertEqual([1.0], v1.eval()) class VariableContainerTest(test.TestCase): diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 7a46157739..2d6a767fed 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -2395,11 +2395,50 @@ class PartitionedVariable(object): def _get_partitions(self): return self._partitions - def assign(self, value, use_locking=False): - _ = value, use_locking - raise NotImplementedError( - "assign() has not been implemented for PartitionedVariable.") + def _apply_assign_fn(self, + assign_fn, + value): + partition_axes = self._partition_axes() + if len(partition_axes) > 1: + raise NotImplementedError( + "Cannot concatenate along more than one dimension: %s. " + "Multi-axis partition assign_fn is not supported" % str(partition_axes)) + partition_ix = partition_axes[0] + size_splits_list = [ + var.shape[partition_ix].value for var in self._variable_list] + value_list = array_ops.split( + value, size_splits_list, axis=partition_ix) + op_list = [ + assign_fn(var, value_list[idx], idx) \ + for idx, var in enumerate(self._variable_list)] + return op_list + def assign(self, value, use_locking=False, name=None, read_value=True): + assign_fn = lambda var, r_value, idx: var.assign( + r_value, use_locking=use_locking, + name="%s_%d" % (name, idx), read_value=read_value) + assign_list = self._apply_assign_fn(assign_fn, value) + if read_value: + return assign_list + return [assign.op for assign in assign_list] + + def assign_add(self, value, use_locking=False, name=None, read_value=True): + assign_fn = lambda var, r_value, idx: var.assign_add( + r_value, use_locking=use_locking, + name="%s_%d" % (name, idx), read_value=read_value) + assign_list = self._apply_assign_fn(assign_fn, value) + if read_value: + return assign_list + return [assign.op for assign in assign_list] + + def assign_sub(self, value, use_locking=False, name=None, read_value=True): + assign_fn = lambda var, r_value, idx: var.assign_sub( + r_value, use_locking=use_locking, + name="%s_%d" % (name, idx), read_value=read_value) + assign_list = self._apply_assign_fn(assign_fn, value) + if read_value: + return assign_list + return [assign.op for assign in assign_list] @tf_export("global_variables") def global_variables(scope=None): -- GitLab From a4eecdb369ecdae3b7fe7c1415d7b3b55bcc7b9e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 23 Sep 2018 17:14:53 +0000 Subject: [PATCH 0079/1085] Fix GPU build issue on python 3 Signed-off-by: Yong Tang --- tensorflow/contrib/image/kernels/image_ops.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tensorflow/contrib/image/kernels/image_ops.h b/tensorflow/contrib/image/kernels/image_ops.h index 6b63eed130..7fac774d07 100644 --- a/tensorflow/contrib/image/kernels/image_ops.h +++ b/tensorflow/contrib/image/kernels/image_ops.h @@ -71,14 +71,7 @@ class ProjectiveGenerator { (transform[3] * output_x + transform[4] * output_y + transform[5]) / projection; - // TODO(ringwalt): Add a fill value input. -#if (defined __CUDA_ARCH__) && (CUDART_VERSION < 8000) - // On CUDA versions previous to 8.0, only __shared__ variables - // could be declared as static in the device code. const T fill_value = T(0); -#else - static const T fill_value = T(0); -#endif switch (interpolation_) { case INTERPOLATION_NEAREST: // Switch the order of x and y again for indexing into the image. -- GitLab From 8f4ded5884684f40b4912d95c717b185340996b8 Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Mon, 24 Sep 2018 11:07:21 +0300 Subject: [PATCH 0080/1085] Fix clang styles. --- tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h index 484cc4d6f5..6753c67701 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h @@ -24,9 +24,7 @@ constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; class ByteSwapper { public: - ByteSwapper(bool big_endian) { - swap_ = big_endian == kLittleEndian; - } + ByteSwapper(bool big_endian) { swap_ = big_endian == kLittleEndian; } inline void SwapIfRequiredInt16(int16_t *x) const { if (swap_) { -- GitLab From 90c68770467701a23d23a85c5d769f6f4fa39f0f Mon Sep 17 00:00:00 2001 From: Anton Dmitriev Date: Mon, 24 Sep 2018 12:14:45 +0300 Subject: [PATCH 0081/1085] Fix byte-order issue. --- tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h index 6753c67701..46df3e39dc 100644 --- a/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h +++ b/tensorflow/contrib/ignite/kernels/ignite_byte_swapper.h @@ -17,14 +17,13 @@ limitations under the License. #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGNITE_BYTE_SWAPPER_H_ #include +#include "tensorflow/core/platform/byte_order.h" namespace tensorflow { -constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; - class ByteSwapper { public: - ByteSwapper(bool big_endian) { swap_ = big_endian == kLittleEndian; } + ByteSwapper(bool big_endian) { swap_ = big_endian == port::kLittleEndian; } inline void SwapIfRequiredInt16(int16_t *x) const { if (swap_) { -- GitLab From ef2c1190d6dc7ec8bca911d03ca2c67b8692a293 Mon Sep 17 00:00:00 2001 From: "Dougal J. Sutherland" Date: Mon, 23 Jul 2018 19:22:57 +0100 Subject: [PATCH 0082/1085] add KID implementation --- tensorflow/contrib/gan/README.md | 2 +- .../eval/python/classifier_metrics_impl.py | 380 +++++++++++++++++- .../eval/python/classifier_metrics_test.py | 97 +++++ 3 files changed, 476 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md index 4ead66ca13..6a4ead86fe 100644 --- a/tensorflow/contrib/gan/README.md +++ b/tensorflow/contrib/gan/README.md @@ -47,7 +47,7 @@ Easily experiment with already-implemented and well-tested losses and penalties, such as the Wasserstein loss, gradient penalty, mutual information penalty, etc * [evaluation](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/eval/python/): -Use `Inception Score` or `Frechet Distance` with a pretrained Inception +Use `Inception Score`, `Frechet Distance`, or `Kernel Distance` with a pretrained Inception network to evaluate your unconditional generative model. You can also use your own pretrained classifier for more specific performance numbers, or use other methods for evaluating conditional generative models. diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index d914f54945..7dc60df474 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -14,8 +14,8 @@ # ============================================================================== """Model evaluation tools for TFGAN. -These methods come from https://arxiv.org/abs/1606.03498 and -https://arxiv.org/abs/1706.08500. +These methods come from https://arxiv.org/abs/1606.03498, +https://arxiv.org/abs/1706.08500, and https://arxiv.org/abs/1801.01401. NOTE: This implementation uses the same weights as in https://github.com/openai/improved-gan/blob/master/inception_score/model.py, @@ -40,6 +40,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import importer from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import image_ops from tensorflow.python.ops import linalg_ops @@ -64,6 +65,12 @@ __all__ = [ 'frechet_classifier_distance_from_activations', 'mean_only_frechet_classifier_distance_from_activations', 'diagonal_only_frechet_classifier_distance_from_activations', + 'kernel_inception_distance', + 'kernel_inception_distance_and_std', + 'kernel_classifier_distance', + 'kernel_classifier_distance_and_std', + 'kernel_classifier_distance_from_activations', + 'kernel_classifier_distance_and_std_from_activations', 'INCEPTION_DEFAULT_IMAGE_SIZE', ] @@ -734,3 +741,372 @@ frechet_inception_distance = functools.partial( frechet_classifier_distance, classifier_fn=functools.partial( run_inception, output_tensor=INCEPTION_FINAL_POOL)) + + +def kernel_classifier_distance(real_images, + generated_images, + classifier_fn, + num_classifier_batches=1, + max_block_size=1024, + dtype=None): + """Kernel "classifier" distance for evaluating a generative model. + + This is based on the Kernel Inception distance, but for an arbitrary + embedding. + + This technique is described in detail in https://arxiv.org/abs/1801.01401. + Given two distributions P and Q of activations, this function calculates + + E_{X, X' ~ P}[k(X, X')] + E_{Y, Y' ~ Q}[k(Y, Y')] + - 2 E_{X ~ P, Y ~ Q}[k(X, Y)] + + where k is the polynomial kernel + + k(x, y) = ( x^T y / dimension + 1 )^3. + + This captures how different the distributions of real and generated images' + visual features are. Like the Frechet distance (and unlike the Inception + score), this is a true distance and incorporates information about the + target images. Unlike the Frechet score, this function computes an + *unbiased* and asymptotically normal estimator, which makes comparing + estimates across models much more intuitive. + + The estimator used takes time quadratic in max_block_size. Larger values of + max_block_size will decrease the variance of the estimator but increase the + computational cost. This differs slightly from the estimator used by the + original paper; it is the block estimator of https://arxiv.org/abs/1307.1954. + + NOTE: the blocking code assumes that real_activations and + generated_activations are both in random order. If either is sorted in a + meaningful order, the estimator will behave poorly. + + NOTE: This function consumes images, computes their activations, and then + computes the classifier score. If you would like to precompute many + activations for real and generated images for large batches, or to compute + multiple scores based on the same images, please use + kernel_clasifier_distance_from_activations(), which this method also uses. + + Args: + real_images: Real images to use to compute Kernel Inception distance. + generated_images: Generated images to use to compute Kernel Inception + distance. + classifier_fn: A function that takes images and produces activations + based on a classifier. + num_classifier_batches: Number of batches to split images in to in order to + efficiently run them through the classifier network. + max_estimator_block_size: integer, default 1024. The distance estimator + splits samples into blocks for computational efficiency. Larger values + are more computationally expensive but decrease the variance of the + distance estimate. + dtype: if not None, coerce activations to this dtype before computations. + + Returns: + The Kernel Inception Distance. A floating-point scalar of the same type + as the output of the activations. + """ + return kernel_classifier_distance_and_std( + real_images, generated_images, classifier_fn, + num_classifier_batches=num_classifier_batches, + max_block_size=max_block_size, + dtype=dtype)[0] + + +kernel_inception_distance = functools.partial( + kernel_classifier_distance, + classifier_fn=functools.partial( + run_inception, output_tensor=INCEPTION_FINAL_POOL)) + + +def kernel_classifier_distance_and_std(real_images, + generated_images, + classifier_fn, + num_classifier_batches=1, + max_block_size=1024, + dtype=None): + """Kernel "classifier" distance for evaluating a generative model. + + This is based on the Kernel Inception distance, but for an arbitrary + embedding. Also returns an estimate of the standard error of the distance + estimator. + + This technique is described in detail in https://arxiv.org/abs/1801.01401. + Given two distributions P and Q of activations, this function calculates + + E_{X, X' ~ P}[k(X, X')] + E_{Y, Y' ~ Q}[k(Y, Y')] + - 2 E_{X ~ P, Y ~ Q}[k(X, Y)] + + where k is the polynomial kernel + + k(x, y) = ( x^T y / dimension + 1 )^3. + + This captures how different the distributions of real and generated images' + visual features are. Like the Frechet distance (and unlike the Inception + score), this is a true distance and incorporates information about the + target images. Unlike the Frechet score, this function computes an + *unbiased* and asymptotically normal estimator, which makes comparing + estimates across models much more intuitive. + + The estimator used takes time quadratic in max_block_size. Larger values of + max_block_size will decrease the variance of the estimator but increase the + computational cost. This differs slightly from the estimator used by the + original paper; it is the block estimator of https://arxiv.org/abs/1307.1954. + + NOTE: the blocking code assumes that real_activations and + generated_activations are both in random order. If either is sorted in a + meaningful order, the estimator will behave poorly. + + NOTE: This function consumes images, computes their activations, and then + computes the classifier score. If you would like to precompute many + activations for real and generated images for large batches, or to compute + multiple scores based on the same images, please use + kernel_clasifier_distance_from_activations(), which this method also uses. + + Args: + real_images: Real images to use to compute Kernel Inception distance. + generated_images: Generated images to use to compute Kernel Inception + distance. + classifier_fn: A function that takes images and produces activations + based on a classifier. + num_classifier_batches: Number of batches to split images in to in order to + efficiently run them through the classifier network. + max_estimator_block_size: integer, default 1024. The distance estimator + splits samples into blocks for computational efficiency. Larger values + are more computationally expensive but decrease the variance of the + distance estimate. Having a smaller block size also gives a better + estimate of the standard error. + dtype: if not None, coerce activations to this dtype before computations. + + Returns: + The Kernel Inception Distance. A floating-point scalar of the same type + as the output of the activations. + An estimate of the standard error of the distance estimator (a scalar of + the same type). + """ + real_images_list = array_ops.split( + real_images, num_or_size_splits=num_classifier_batches) + generated_images_list = array_ops.split( + generated_images, num_or_size_splits=num_classifier_batches) + + real_imgs = array_ops.stack(real_images_list) + generated_imgs = array_ops.stack(generated_images_list) + + # Compute the activations using the memory-efficient `map_fn`. + def compute_activations(elems): + return functional_ops.map_fn(fn=classifier_fn, + elems=elems, + parallel_iterations=1, + back_prop=False, + swap_memory=True, + name='RunClassifier') + + real_a = compute_activations(real_imgs) + gen_a = compute_activations(generated_imgs) + + # Ensure the activations have the right shapes. + real_a = array_ops.concat(array_ops.unstack(real_a), 0) + gen_a = array_ops.concat(array_ops.unstack(gen_a), 0) + + return kernel_classifier_distance_and_std_from_activations( + real_a, gen_a, max_block_size=max_block_size) + + +kernel_inception_distance_and_std = functools.partial( + kernel_classifier_distance_and_std, + classifier_fn=functools.partial( + run_inception, output_tensor=INCEPTION_FINAL_POOL)) + + +def kernel_classifier_distance_from_activations(real_activations, + generated_activations, + max_block_size=1024, + dtype=None): + '''Kernel "classifier" distance for evaluating a generative model. + + This methods computes the kernel classifier distance from activations of + real images and generated images. This can be used independently of the + kernel_classifier_distance() method, especially in the case of using large + batches during evaluation where we would like to precompute all of the + activations before computing the classifier distance, or if we want to + compute multiple metrics based on the same images. + + This technique is described in detail in https://arxiv.org/abs/1801.01401. + Given two distributions P and Q of activations, this function calculates + + E_{X, X' ~ P}[k(X, X')] + E_{Y, Y' ~ Q}[k(Y, Y')] + - 2 E_{X ~ P, Y ~ Q}[k(X, Y)] + + where k is the polynomial kernel + + k(x, y) = ( x^T y / dimension + 1 )^3. + + This captures how different the distributions of real and generated images' + visual features are. Like the Frechet distance (and unlike the Inception + score), this is a true distance and incorporates information about the + target images. Unlike the Frechet score, this function computes an + *unbiased* and asymptotically normal estimator, which makes comparing + estimates across models much more intuitive. + + The estimator used takes time quadratic in max_block_size. Larger values of + max_block_size will decrease the variance of the estimator but increase the + computational cost. This differs slightly from the estimator used by the + original paper; it is the block estimator of https://arxiv.org/abs/1307.1954. + + NOTE: the blocking code assumes that real_activations and + generated_activations are both in random order. If either is sorted in a + meaningful order, the estimator will behave poorly. + + Args: + real_activations: 2D Tensor containing activations of real data. Shape is + [batch_size, activation_size]. + generated_activations: 2D Tensor containing activations of generated data. + Shape is [batch_size, activation_size]. + max_block_size: integer, default 1024. The distance estimator + splits samples into blocks for computational efficiency. Larger values + are more computationally expensive but decrease the variance of the + distance estimate. + dtype: if not None, coerce activations to this dtype before computations. + + Returns: + The Kernel Inception Distance. A floating-point scalar of the same type + as the output of the activations. + ''' + return kernel_classifier_distance_and_std_from_activations( + real_activations, generated_activations, + max_block_size=max_block_size)[0] + + +def kernel_classifier_distance_and_std_from_activations(real_activations, + generated_activations, + max_block_size=1024, + dtype=None): + '''Kernel "classifier" distance for evaluating a generative model. + + This methods computes the kernel classifier distance from activations of + real images and generated images. This can be used independently of the + kernel_classifier_distance() method, especially in the case of using large + batches during evaluation where we would like to precompute all of the + activations before computing the classifier distance, or if we want to + compute multiple metrics based on the same images. It also returns a rough + estimate of the standard error of the estimator. + + This technique is described in detail in https://arxiv.org/abs/1801.01401. + Given two distributions P and Q of activations, this function calculates + + E_{X, X' ~ P}[k(X, X')] + E_{Y, Y' ~ Q}[k(Y, Y')] + - 2 E_{X ~ P, Y ~ Q}[k(X, Y)] + + where k is the polynomial kernel + + k(x, y) = ( x^T y / dimension + 1 )^3. + + This captures how different the distributions of real and generated images' + visual features are. Like the Frechet distance (and unlike the Inception + score), this is a true distance and incorporates information about the + target images. Unlike the Frechet score, this function computes an + *unbiased* and asymptotically normal estimator, which makes comparing + estimates across models much more intuitive. + + The estimator used takes time quadratic in max_block_size. Larger values of + max_block_size will decrease the variance of the estimator but increase the + computational cost. This differs slightly from the estimator used by the + original paper; it is the block estimator of https://arxiv.org/abs/1307.1954. + The estimate of the standard error will also be more reliable when there are + more blocks, i.e. when max_block_size is smaller. + + NOTE: the blocking code assumes that real_activations and + generated_activations are both in random order. If either is sorted in a + meaningful order, the estimator will behave poorly. + + Args: + real_activations: 2D Tensor containing activations of real data. Shape is + [batch_size, activation_size]. + generated_activations: 2D Tensor containing activations of generated data. + Shape is [batch_size, activation_size]. + max_block_size: integer, default 1024. The distance estimator + splits samples into blocks for computational efficiency. Larger values + are more computationally expensive but decrease the variance of the + distance estimate. Having a smaller block size also gives a better + estimate of the standard error. + dtype: if not None, coerce activations to this dtype before computations. + + Returns: + The Kernel Inception Distance. A floating-point scalar of the same type + as the output of the activations. + An estimate of the standard error of the distance estimator (a scalar of + the same type). + ''' + + real_activations.shape.assert_has_rank(2) + generated_activations.shape.assert_has_rank(2) + real_activations.shape[1].assert_is_compatible_with( + generated_activations.shape[1]) + + if dtype is None: + dtype = real_activations.dtype + assert generated_activations.dtype == dtype + else: + real_activations = math_ops.cast(real_activations, dtype) + generated_activations = math_ops.cast(generated_activations, dtype) + + # Figure out how to split the activations into blocks of approximately + # equal size, with none larger than max_block_size. + n_r = array_ops.shape(real_activations)[0] + n_g = array_ops.shape(generated_activations)[0] + + n_bigger = math_ops.maximum(n_r, n_g) + n_blocks = math_ops.to_int32(math_ops.ceil(n_bigger / max_block_size)) + + v_r = n_r // n_blocks + v_g = n_g // n_blocks + + n_plusone_r = n_r - v_r * n_blocks + n_plusone_g = n_g - v_g * n_blocks + + sizes_r = array_ops.concat([ + array_ops.fill([n_blocks - n_plusone_r], v_r), + array_ops.fill([n_plusone_r], v_r + 1), + ], 0) + sizes_g = array_ops.concat([ + array_ops.fill([n_blocks - n_plusone_g], v_g), + array_ops.fill([n_plusone_g], v_g + 1), + ], 0) + + zero = array_ops.zeros([1], dtype=dtypes.int32) + inds_r = array_ops.concat([zero, math_ops.cumsum(sizes_r)], 0) + inds_g = array_ops.concat([zero, math_ops.cumsum(sizes_g)], 0) + + dim = math_ops.cast(real_activations.shape[1], dtype) + + def compute_kid_block(i): + "Compute the ith block of the KID estimate." + r_s = inds_r[i] + r_e = inds_r[i + 1] + r = real_activations[r_s:r_e] + m = math_ops.cast(r_e - r_s, dtype) + + g_s = inds_g[i] + g_e = inds_g[i + 1] + g = generated_activations[g_s:g_e] + n = math_ops.cast(g_e - g_s, dtype) + + k_rr = (math_ops.matmul(r, r, transpose_b=True) / dim + 1) ** 3 + k_rg = (math_ops.matmul(r, g, transpose_b=True) / dim + 1) ** 3 + k_gg = (math_ops.matmul(g, g, transpose_b=True) / dim + 1) ** 3 + return ( + -2 * math_ops.reduce_mean(k_rg) + + (math_ops.reduce_sum(k_rr) - math_ops.trace(k_rr)) / (m * (m - 1)) + + (math_ops.reduce_sum(k_gg) - math_ops.trace(k_gg)) / (n * (n - 1))) + + ests = functional_ops.map_fn( + compute_kid_block, math_ops.range(n_blocks), dtype=dtype, back_prop=False) + + mn = math_ops.reduce_mean(ests) + + # nn_impl.moments doesn't use the Bessel correction, which we want here + n_blocks_ = math_ops.cast(n_blocks, dtype) + var = control_flow_ops.cond( + math_ops.less_equal(n_blocks, 1), + lambda: array_ops.constant(float("nan"), dtype=dtype), + lambda: math_ops.reduce_sum(math_ops.square(ests - mn)) / (n_blocks_ - 1)) + + return mn, math_ops.sqrt(var / n_blocks_) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py index 4fb8d58bc9..b6042bdb72 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py @@ -86,6 +86,43 @@ def _expected_fid(real_imgs, gen_imgs): def _expected_trace_sqrt_product(sigma, sigma_v): return np.trace(scp_linalg.sqrtm(np.dot(sigma, sigma_v))) + +def _expected_kid_and_std(real_imgs, gen_imgs, max_block_size=1024): + n_r, dim = real_imgs.shape + n_g = gen_imgs.shape[0] + + n_blocks = int(np.ceil(max(n_r, n_g) / max_block_size)) + + sizes_r = np.full(n_blocks, n_r // n_blocks) + to_patch = n_r - n_blocks * (n_r // n_blocks) + if to_patch > 0: + sizes_r[-to_patch:] += 1 + inds_r = np.r_[0, np.cumsum(sizes_r)] + assert inds_r[-1] == n_r + + sizes_g = np.full(n_blocks, n_g // n_blocks) + to_patch = n_g - n_blocks * (n_g // n_blocks) + if to_patch > 0: + sizes_g[-to_patch:] += 1 + inds_g = np.r_[0, np.cumsum(sizes_g)] + assert inds_g[-1] == n_g + + ests = [] + for i in range(n_blocks): + r = real_imgs[inds_r[i]:inds_r[i + 1]] + g = gen_imgs[inds_g[i]:inds_g[i + 1]] + + k_rr = (np.dot(r, r.T) / dim + 1) ** 3 + k_rg = (np.dot(r, g.T) / dim + 1) ** 3 + k_gg = (np.dot(g, g.T) / dim + 1) ** 3 + ests.append( + -2 * k_rg.mean() + + k_rr[np.triu_indices_from(k_rr, k=1)].mean() + + k_gg[np.triu_indices_from(k_gg, k=1)].mean()) + + var = np.var(ests, ddof=1) if len(ests) > 1 else np.nan + return np.mean(ests), np.sqrt(var / len(ests)) + # A dummy GraphDef string with the minimum number of Ops. graphdef_string = """ node { @@ -272,6 +309,18 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase): # Check that none of the model variables are trainable. self.assertListEqual([], variables.trainable_variables()) + def test_kernel_inception_distance_graph(self): + """Test `frechet_inception_distance` graph construction.""" + img = array_ops.ones([7, 299, 299, 3]) + distance = _run_with_mock( + classifier_metrics.kernel_inception_distance, img, img) + + self.assertTrue(isinstance(distance, ops.Tensor)) + distance.shape.assert_has_rank(0) + + # Check that none of the model variables are trainable. + self.assertListEqual([], variables.trainable_variables()) + def test_run_inception_multicall(self): """Test that `run_inception` can be called multiple times.""" for batch_size in (7, 3, 2): @@ -411,6 +460,54 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase): # Check that the FIDs increase monotonically. self.assertTrue(all(fid_a < fid_b for fid_a, fid_b in zip(fids, fids[1:]))) + def test_kernel_classifier_distance_value(self): + """Test that `kernel_classifier_distance` gives the correct value.""" + np.random.seed(0) + + test_pool_real_a = np.float32(np.random.randn(512, 256)) + test_pool_gen_a = np.float32(np.random.randn(512, 256) * 1.1 + .05) + + kid_op = _run_with_mock( + classifier_metrics.kernel_classifier_distance_and_std, + test_pool_real_a, + test_pool_gen_a, + classifier_fn=lambda x: x, + max_block_size=600) + + with self.test_session() as sess: + actual_kid, actual_std = sess.run(kid_op) + + expected_kid, expected_std = _expected_kid_and_std( + test_pool_real_a, test_pool_gen_a) + + self.assertAllClose(expected_kid, actual_kid, 0.001) + self.assertAllClose(expected_std, actual_std, 0.001) + + def test_kernel_classifier_distance_block_sizes(self): + """Test that `kernel_classifier_distance` works with unusual max_block_size + values..""" + np.random.seed(0) + + test_pool_real_a = np.float32(np.random.randn(512, 256)) + test_pool_gen_a = np.float32(np.random.randn(768, 256) * 1.1 + .05) + + max_block_size = array_ops.placeholder(dtypes.int32, shape=()) + kid_op = _run_with_mock( + classifier_metrics.kernel_classifier_distance_and_std_from_activations, + array_ops.constant(test_pool_real_a), + array_ops.constant(test_pool_gen_a), + max_block_size=max_block_size) + + for block_size in [50, 512, 1000]: + with self.test_session() as sess: + actual_kid, actual_std = sess.run(kid_op, {max_block_size: block_size}) + + expected_kid, expected_std = _expected_kid_and_std( + test_pool_real_a, test_pool_gen_a, max_block_size=block_size) + + self.assertAllClose(expected_kid, actual_kid, 0.001) + self.assertAllClose(expected_std, actual_std, 0.001) + def test_trace_sqrt_product_value(self): """Test that `trace_sqrt_product` gives the correct value.""" np.random.seed(0) -- GitLab From f0886f7269de900d226455d4831722f6fc94a71b Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Tue, 25 Sep 2018 09:59:17 +0800 Subject: [PATCH 0083/1085] Fix build dependencies in tensorflow/cc/BUILD. --- tensorflow/cc/BUILD | 1 + tensorflow/python/kernel_tests/relu_op_test.py | 4 ++-- tensorflow/python/ops/nn_ops.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD index f56521dac0..e99d15f85d 100644 --- a/tensorflow/cc/BUILD +++ b/tensorflow/cc/BUILD @@ -410,6 +410,7 @@ tf_cc_test( srcs = ["gradients/nn_grad_test.cc"], deps = [ ":cc_ops", + ":cc_ops_internal", ":grad_op_registry", ":grad_testutil", ":gradient_checker", diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index 86d9c90e83..d97a1613b9 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -351,7 +351,7 @@ class LeakyReluTest(test.TestCase): self.assertLess(err, 1e-10) def testGradGradFloat32(self): - with compat.forward_compatibility_horizon(2018, 10, 2): + with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], @@ -369,7 +369,7 @@ class LeakyReluTest(test.TestCase): self.assertLess(err, 1e-4) def testGradGradFloat64(self): - with compat.forward_compatibility_horizon(2018, 10, 2): + with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index d646245ce3..2861f40586 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1601,7 +1601,7 @@ def leaky_relu(features, alpha=0.2, name=None): features = ops.convert_to_tensor(features, name="features") if features.dtype.is_integer: features = math_ops.to_float(features) - if compat.forward_compatible(2018, 10, 1): + if compat.forward_compatible(2018, 11, 1): return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name) alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha") return math_ops.maximum(alpha * features, features, name=name) -- GitLab From c12a90e45c5f94b80289f4278f81be4a0348fa19 Mon Sep 17 00:00:00 2001 From: wangsiyu Date: Tue, 25 Sep 2018 13:51:36 +0800 Subject: [PATCH 0084/1085] fix pylint --- tensorflow/python/ops/variables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 2d6a767fed..d058478d58 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -2402,7 +2402,8 @@ class PartitionedVariable(object): if len(partition_axes) > 1: raise NotImplementedError( "Cannot concatenate along more than one dimension: %s. " - "Multi-axis partition assign_fn is not supported" % str(partition_axes)) + "Multi-axis partition assign_fn is not supported " + % str(partition_axes)) partition_ix = partition_axes[0] size_splits_list = [ var.shape[partition_ix].value for var in self._variable_list] -- GitLab From 3d60d636de59449a8448cbcbcd71af82e2871538 Mon Sep 17 00:00:00 2001 From: wangsiyu Date: Tue, 25 Sep 2018 13:53:36 +0800 Subject: [PATCH 0085/1085] fix back variabe name --- tensorflow/python/kernel_tests/variables_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py index 687784c8b7..0b101529fe 100644 --- a/tensorflow/python/kernel_tests/variables_test.py +++ b/tensorflow/python/kernel_tests/variables_test.py @@ -673,7 +673,7 @@ class PartitionedVariableTest(test.TestCase): v0._set_save_slice_info( variables.Variable.SaveSliceInfo(v0.name, [2], [0], [1])) v1._set_save_slice_info( - variables.Variable.SaveSliceInfo(v1.name, [2], [1], [1])) + variables.Variable.SaveSliceInfo(v0.name, [2], [1], [1])) partitions = [2] variables.PartitionedVariable( -- GitLab From 21d4e8bb30a1753a81edd4912881d95b47ae3d1c Mon Sep 17 00:00:00 2001 From: wangsiyu Date: Tue, 25 Sep 2018 15:50:10 +0800 Subject: [PATCH 0086/1085] remove warning lines --- tensorflow/python/ops/variables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index d058478d58..69f63bc8e6 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -2401,7 +2401,6 @@ class PartitionedVariable(object): partition_axes = self._partition_axes() if len(partition_axes) > 1: raise NotImplementedError( - "Cannot concatenate along more than one dimension: %s. " "Multi-axis partition assign_fn is not supported " % str(partition_axes)) partition_ix = partition_axes[0] -- GitLab From 7630e9df4804a01f5dd0ab20d4c0bcfb58e45432 Mon Sep 17 00:00:00 2001 From: Richard Yu Date: Tue, 25 Sep 2018 15:50:13 -0700 Subject: [PATCH 0087/1085] Fixing error --- tensorflow/contrib/quantize/python/fold_batch_norms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index d882b79892..d9f179bee4 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -628,7 +628,7 @@ def _GetBatchNormParams(graph, context, has_scaling): bn_decay_var_tensor = _FindMatchingTensor(graph, op_suffix_bn_decay_var, context) if batch_mean_tensor is None and moving_mean_tensor is None: - raise ValueError('Error folding unfused batch norms') + ValueError('Error folding unfused batch norms') if has_scaling: gamma_tensor = _FindMatchingTensor(graph, op_suffix_gamma, context) -- GitLab From f55e5ef27b3ccf1b75932e219f7358976dbf56c2 Mon Sep 17 00:00:00 2001 From: IMBurbank Date: Tue, 25 Sep 2018 18:39:11 -0600 Subject: [PATCH 0088/1085] Update to use python 2-3 compatible function tf_inspect.getfullargspec. --- .../python/losses/python/tuple_losses_impl.py | 2 +- .../labeled_tensor/python/ops/_typecheck.py | 2 +- .../layers/python/layers/rev_block_lib.py | 3 +- .../python/learn/estimators/estimator.py | 4 +- .../learn/python/learn/estimators/head.py | 2 +- .../learn/python/learn/experiment_test.py | 2 +- .../learn/python/learn/export_strategy.py | 2 +- .../contrib/learn/python/learn/metric_spec.py | 2 +- .../contrib/learn/python/learn/monitors.py | 2 +- .../contrib/tpu/python/tpu/tpu_function.py | 2 +- tensorflow/python/framework/errors_impl.py | 2 +- tensorflow/python/framework/function.py | 6 +- tensorflow/python/keras/backend_test.py | 2 +- tensorflow/python/keras/testing_utils.py | 2 +- .../kernel_tests/variable_scope_test.py | 4 +- tensorflow/python/ops/variable_scope.py | 4 +- tensorflow/python/util/tf_contextlib_test.py | 2 +- tensorflow/python/util/tf_inspect.py | 7 +- tensorflow/python/util/tf_inspect_test.py | 249 +++++++++++++++++- .../api/lib/python_object_to_proto_visitor.py | 2 +- 20 files changed, 267 insertions(+), 36 deletions(-) diff --git a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py index 221c70c38b..00a83e5e55 100644 --- a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py +++ b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py @@ -101,7 +101,7 @@ def _args_to_gan_model(loss_fn): """ # Match arguments in `loss_fn` to elements of `namedtuple`. # TODO(joelshor): Properly handle `varargs` and `keywords`. - argspec = tf_inspect.getargspec(loss_fn) + argspec = tf_inspect.getfullargspec(loss_fn) defaults = argspec.defaults or [] required_args = set(argspec.args[:-len(defaults)]) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py b/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py index 80fa17ec1f..0e23039847 100644 --- a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py +++ b/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py @@ -230,7 +230,7 @@ def accepts(*types): def check_accepts(f): """Check the types.""" - spec = tf_inspect.getargspec(f) + spec = tf_inspect.getfullargspec(f) num_function_arguments = len(spec.args) if len(types) != num_function_arguments: diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py index 06da32072f..55979cc391 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py @@ -576,7 +576,8 @@ def _recomputing_grad_fn(compute_fn, def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): """See recompute_grad.""" - has_is_recompute_kwarg = "is_recomputing" in tf_inspect.getargspec(fn).args + has_is_recompute_kwarg = ( + "is_recomputing" in tf_inspect.getfullargspec(fn).args) for arg in args: if not isinstance(arg, framework_ops.Tensor): raise ValueError("All inputs to function must be Tensors") diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index c1de42782e..b88923bca2 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -199,11 +199,11 @@ def _model_fn_args(fn): if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): # Handle functools.partial and similar objects. return tuple([ - arg for arg in tf_inspect.getargspec(fn.func).args[len(fn.args):] + arg for arg in tf_inspect.getfullargspec(fn.func).args[len(fn.args):] if arg not in set(fn.keywords.keys()) ]) # Handle function. - return tuple(tf_inspect.getargspec(fn).args) + return tuple(tf_inspect.getfullargspec(fn).args) def _get_replica_device_setter(config): diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index c6f79e00d5..63dd08316b 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -1861,7 +1861,7 @@ def _get_arguments(func): _, func = tf_decorator.unwrap(func) if hasattr(func, "__code__"): # Regular function. - return tf_inspect.getargspec(func) + return tf_inspect.getfullargspec(func) elif hasattr(func, "func"): # Partial function. return _get_arguments(func.func) diff --git a/tensorflow/contrib/learn/python/learn/experiment_test.py b/tensorflow/contrib/learn/python/learn/experiment_test.py index fb16c94c29..6926696fb6 100644 --- a/tensorflow/contrib/learn/python/learn/experiment_test.py +++ b/tensorflow/contrib/learn/python/learn/experiment_test.py @@ -126,7 +126,7 @@ class TestBaseEstimator(object): def _check_method_supports_args(method, kwargs): """Checks that the given method supports the given args.""" - supported_args = tuple(tf_inspect.getargspec(method).args) + supported_args = tuple(tf_inspect.getfullargspec(method).args) for kwarg in kwargs: if kwarg not in supported_args: raise ValueError( diff --git a/tensorflow/contrib/learn/python/learn/export_strategy.py b/tensorflow/contrib/learn/python/learn/export_strategy.py index 075cab536e..0d6e0cdc18 100644 --- a/tensorflow/contrib/learn/python/learn/export_strategy.py +++ b/tensorflow/contrib/learn/python/learn/export_strategy.py @@ -96,7 +96,7 @@ class ExportStrategy( """ # don't break existing export_fns that don't accept checkpoint_path and # eval_result - export_fn_args = tf_inspect.getargspec(self.export_fn).args + export_fn_args = tf_inspect.getfullargspec(self.export_fn).args kwargs = {} if 'checkpoint_path' in export_fn_args: kwargs['checkpoint_path'] = checkpoint_path diff --git a/tensorflow/contrib/learn/python/learn/metric_spec.py b/tensorflow/contrib/learn/python/learn/metric_spec.py index 97220365d5..604d6d46b4 100644 --- a/tensorflow/contrib/learn/python/learn/metric_spec.py +++ b/tensorflow/contrib/learn/python/learn/metric_spec.py @@ -51,7 +51,7 @@ def _args(fn): return tuple( [arg for arg in _args(fn.func) if arg not in set(fn.keywords.keys())]) # Handle function. - return tuple(tf_inspect.getargspec(fn).args) + return tuple(tf_inspect.getfullargspec(fn).args) _CANONICAL_LABELS_ARG = 'labels' diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py index 3d691d4340..5f61e0264f 100644 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ b/tensorflow/contrib/learn/python/learn/monitors.py @@ -1303,7 +1303,7 @@ class RunHookAdapterForMonitors(session_run_hook.SessionRunHook): def end(self, session): self._last_step = None for m in self._monitors: - if "session" in tf_inspect.getargspec(m.end).args: + if "session" in tf_inspect.getfullargspec(m.end).args: m.end(session=session) else: m.end() diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_function.py b/tensorflow/contrib/tpu/python/tpu/tpu_function.py index 0c7a38dbbb..9c4bd1c4d1 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_function.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_function.py @@ -80,7 +80,7 @@ def check_function_argument_count(func, input_arity, infeed_queue): number_of_arguments_needed = input_arity if infeed_queue is not None: number_of_arguments_needed += infeed_queue.number_of_tuple_elements - arg_spec = tf_inspect.getargspec(func) + arg_spec = tf_inspect.getfullargspec(func) number_of_args = len(arg_spec.args) if arg_spec.defaults is None: number_of_defaults = 0 diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py index 5af71f2cfb..c373e75a74 100644 --- a/tensorflow/python/framework/errors_impl.py +++ b/tensorflow/python/framework/errors_impl.py @@ -55,7 +55,7 @@ class OpError(Exception): def __reduce__(self): # Allow the subclasses to accept less arguments in their __init__. - init_argspec = tf_inspect.getargspec(self.__class__.__init__) + init_argspec = tf_inspect.getfullargspec(self.__class__.__init__) args = tuple(getattr(self, arg) for arg in init_argspec.args[1:]) return self.__class__, args diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index f287289bd0..3db6f683c9 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -132,9 +132,9 @@ class Defun(object): raise ValueError("func %s must be callable" % func) # Func should not use kwargs and defaults. - argspec = tf_inspect.getargspec(func) - if argspec.keywords or argspec.defaults: - raise ValueError("Functions with argument defaults or keyword " + argspec = tf_inspect.getfullargspec(func) + if argspec.varkw or argspec.defaults: + raise ValueError("Functions with argument defaults or varkw " "arguments are not supported.") # Computes how many arguments 'func' has. diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py index ab71589940..31191d0d35 100644 --- a/tensorflow/python/keras/backend_test.py +++ b/tensorflow/python/keras/backend_test.py @@ -452,7 +452,7 @@ class BackendLinearAlgebraTest(test.TestCase): compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5), keras_kwargs={'axis': -1}, np_kwargs={'axis': -1}) - if 'keepdims' in tf_inspect.getargspec(keras_op).args: + if 'keepdims' in tf_inspect.getfullargspec(keras_op).args: compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5), keras_kwargs={'axis': 1, diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py index 501b50ba5f..1afaba5653 100644 --- a/tensorflow/python/keras/testing_utils.py +++ b/tensorflow/python/keras/testing_utils.py @@ -102,7 +102,7 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None, layer.set_weights(weights) # test and instantiation from weights - if 'weights' in tf_inspect.getargspec(layer_cls.__init__): + if 'weights' in tf_inspect.getfullargspec(layer_cls.__init__): kwargs['weights'] = weights layer = layer_cls(**kwargs) diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index 401e1ae102..1d0b72b17a 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -998,8 +998,8 @@ class VariableScopeTest(test.TestCase): def testSignatureGetVarVsGetLocalVar(self): """get_{local,}variable() must take the same list of args.""" - arg_names = tf_inspect.getargspec(variable_scope.get_variable)[0] - local_arg_names = tf_inspect.getargspec( + arg_names = tf_inspect.getfullargspec(variable_scope.get_variable)[0] + local_arg_names = tf_inspect.getfullargspec( variable_scope.get_local_variable)[0] self.assertEqual(arg_names, local_arg_names) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index a43676cd70..3cc1eb916d 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -892,14 +892,14 @@ class _VariableStore(object): if shape and shape.is_fully_defined(): init_val = lambda: initializer( # pylint: disable=g-long-lambda shape.as_list(), dtype=dtype, partition_info=partition_info) - elif not tf_inspect.getargspec(initializer).args: + elif not tf_inspect.getfullargspec(initializer).args: init_val = initializer else: raise ValueError("You can only pass an initializer function that " "expects no arguments to its callable when the " "shape is not fully defined. The given initializer " "function expects the following args %s" % - tf_inspect.getargspec(initializer).args) + tf_inspect.getfullargspec(initializer).args) variable_dtype = dtype.base_dtype # Create the variable. diff --git a/tensorflow/python/util/tf_contextlib_test.py b/tensorflow/python/util/tf_contextlib_test.py index 4a5bf388a6..1e921b5ea3 100644 --- a/tensorflow/python/util/tf_contextlib_test.py +++ b/tensorflow/python/util/tf_contextlib_test.py @@ -83,7 +83,7 @@ class TfContextlibTest(test.TestCase): self.assertFalse(isinstance(target, tf_decorator.TFDecorator)) def testGetArgSpecReturnsWrappedArgSpec(self): - argspec = tf_inspect.getargspec(test_params_and_defaults) + argspec = tf_inspect.getfullargspec(test_params_and_defaults) self.assertEqual(['a', 'b', 'c', 'd'], argspec.args) self.assertEqual((2, True, 'hello'), argspec.defaults) diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py index 967c872c2a..234850ac3f 100644 --- a/tensorflow/python/util/tf_inspect.py +++ b/tensorflow/python/util/tf_inspect.py @@ -43,7 +43,12 @@ def currentframe(): def getargspec(obj): - """TFDecorator-aware replacement for inspect.getargspec. + """TFDecorator-aware replacement for `inspect.getargspec`. + + This should not be called from other modules. It is deprecated in python3. + + Use `getfullargspec`. It is a TFDecorator-aware replacement for + `inspect.getfullargspec` compatible with both python2 and python3. Args: obj: A function, partial function, or callable object, possibly diff --git a/tensorflow/python/util/tf_inspect_test.py b/tensorflow/python/util/tf_inspect_test.py index d3b7e4b969..55f88f8fc6 100644 --- a/tensorflow/python/util/tf_inspect_test.py +++ b/tensorflow/python/util/tf_inspect_test.py @@ -122,18 +122,6 @@ class TfInspectTest(test.TestCase): self.assertEqual(argspec, tf_inspect.getargspec(partial_func)) - def testGetFullArgsSpecForPartial(self): - - def func(a, b): - del a, b - - partial_function = functools.partial(func, 1) - argspec = tf_inspect.FullArgSpec( - args=['b'], varargs=None, varkw=None, defaults=None, - kwonlyargs=[], kwonlydefaults=None, annotations={}) - - self.assertEqual(argspec, tf_inspect.getfullargspec(partial_function)) - def testGetArgSpecOnPartialInvalidArgspec(self): """Tests getargspec on partial function that doesn't have valid argspec.""" @@ -303,6 +291,243 @@ class TfInspectTest(test.TestCase): self.assertEqual(argspec, tf_inspect.getargspec(NewClass)) + def testGetFullArgSpecOnDecoratorsThatDontProvideFullArgSpec(self): + argspec = tf_inspect.getfullargspec( + test_decorated_function_with_defaults) + self.assertEqual(['a', 'b', 'c'], argspec.args) + self.assertEqual((2, 'Hello'), argspec.defaults) + + def testGetFullArgSpecOnDecoratorThatChangesFullArgSpec(self): + argspec = tf_inspect.FullArgSpec( + args=['a', 'b', 'c'], + varargs=None, + varkw=None, + defaults=(1, 'hello'), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + decorator = tf_decorator.TFDecorator('', test_undecorated_function, '', + argspec) + self.assertEqual(argspec, tf_inspect.getfullargspec(decorator)) + + def testGetFullArgSpecIgnoresDecoratorsThatDontProvideFullArgSpec(self): + argspec = tf_inspect.FullArgSpec( + args=['a', 'b', 'c'], + varargs=None, + varkw=None, + defaults=(1, 'hello'), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + inner_decorator = tf_decorator.TFDecorator('', test_undecorated_function, + '', argspec) + outer_decorator = tf_decorator.TFDecorator('', inner_decorator) + self.assertEqual(argspec, tf_inspect.getfullargspec(outer_decorator)) + + def testGetFullArgSpecReturnsOutermostDecoratorThatChangesFullArgSpec(self): + outer_argspec = tf_inspect.FullArgSpec( + args=['a'], varargs=None, varkw=None, defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + inner_argspec = tf_inspect.FullArgSpec( + args=['b'], varargs=None, varkw=None, defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + inner_decorator = tf_decorator.TFDecorator('', test_undecorated_function, + '', inner_argspec) + outer_decorator = tf_decorator.TFDecorator('', inner_decorator, '', + outer_argspec) + self.assertEqual(outer_argspec, + tf_inspect.getfullargspec(outer_decorator)) + + def testGetFullArgsSpecForPartial(self): + + def func(a, b): + del a, b + + partial_function = functools.partial(func, 1) + argspec = tf_inspect.FullArgSpec( + args=['b'], varargs=None, varkw=None, defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_function)) + + def testGetFullArgSpecOnPartialInvalidFullArgSpec(self): + """Tests getfullargspec. + + Tests on partial function that doesn't have valid fullargspec. + """ + + def func(m, n, l, k=4): + return 2 * m + l + n * k + + partial_func = functools.partial(func, n=7) + + exception_message = (r"Some arguments \['l'\] do not have default value, " + "but they are positioned after those with default " + "values. This can not be expressed with ArgSpec.") + with self.assertRaisesRegexp(ValueError, exception_message): + tf_inspect.getfullargspec(partial_func) + + def testGetFullArgSpecOnPartialValidFullArgSpec(self): + """Tests getfullargspec on partial function with valid fullargspec.""" + + def func(m, n, l, k=4): + return 2 * m + l + n * k + + partial_func = functools.partial(func, n=7, l=2) + argspec = tf_inspect.FullArgSpec( + args=['m', 'n', 'l', 'k'], + varargs=None, + varkw=None, + defaults=(7, 2, 4), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialNoArgumentsLeft(self): + """Tests getfullargspec on partial function that prunes all arguments.""" + + def func(m, n): + return 2 * m + n + + partial_func = functools.partial(func, 7, 10) + argspec = tf_inspect.FullArgSpec( + args=[], varargs=None, varkw=None, defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialKeywordArgument(self): + """Tests getfullargspec on partial function that prunes some arguments.""" + + def func(m, n): + return 2 * m + n + + partial_func = functools.partial(func, n=7) + argspec = tf_inspect.FullArgSpec( + args=['m', 'n'], varargs=None, varkw=None, defaults=(7,), + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialKeywordArgumentWithDefaultValue(self): + """Tests getfullargspec. + + Tests on partial function that prunes argument by keyword. + """ + + def func(m=1, n=2): + return 2 * m + n + + partial_func = functools.partial(func, n=7) + argspec = tf_inspect.FullArgSpec( + args=['m', 'n'], varargs=None, varkw=None, defaults=(1, 7), + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialWithVarargs(self): + """Tests getfullargspec on partial function with variable arguments.""" + + def func(m, *arg): + return m + len(arg) + + partial_func = functools.partial(func, 7, 8) + argspec = tf_inspect.FullArgSpec( + args=[], varargs='arg', varkw=None, defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialWithVarkwargs(self): + """Tests getfullargspec. + + Tests on partial function with variable keyword arguments. + """ + + def func(m, n, **kwarg): + return m * n + len(kwarg) + + partial_func = functools.partial(func, 7) + argspec = tf_inspect.FullArgSpec( + args=['n'], varargs=None, varkw='kwarg', defaults=None, + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnPartialWithDecorator(self): + """Tests getfullargspec on decorated partial function.""" + + @test_decorator('decorator') + def func(m=1, n=2): + return 2 * m + n + + partial_func = functools.partial(func, n=7) + argspec = tf_inspect.FullArgSpec( + args=['m', 'n'], varargs=None, varkw=None, defaults=(1, 7), + kwonlyargs=[], kwonlydefaults=None, annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) + + def testGetFullArgSpecOnCallableObject(self): + + class Callable(object): + + def __call__(self, a, b=1, c='hello'): + pass + + argspec = tf_inspect.FullArgSpec( + args=['self', 'a', 'b', 'c'], + varargs=None, + varkw=None, + defaults=(1, 'hello'), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + test_obj = Callable() + self.assertEqual(argspec, tf_inspect.getfullargspec(test_obj)) + + def testGetFullArgSpecOnInitClass(self): + + class InitClass(object): + + def __init__(self, a, b=1, c='hello'): + pass + + argspec = tf_inspect.FullArgSpec( + args=['self', 'a', 'b', 'c'], + varargs=None, + varkw=None, + defaults=(1, 'hello'), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(InitClass)) + + def testGetFullArgSpecOnNewClass(self): + + class NewClass(object): + + def __new__(cls, a, b=1, c='hello'): + pass + + argspec = tf_inspect.FullArgSpec( + args=['cls', 'a', 'b', 'c'], + varargs=None, + varkw=None, + defaults=(1, 'hello'), + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + + self.assertEqual(argspec, tf_inspect.getfullargspec(NewClass)) + def testGetDoc(self): self.assertEqual('Test Decorated Function With Defaults Docstring.', tf_inspect.getdoc(test_decorated_function_with_defaults)) diff --git a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py index 3a48cf683c..2a40caf720 100644 --- a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py +++ b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py @@ -47,7 +47,7 @@ def _SanitizedArgSpec(obj): string, a string representation of the argspec. """ output_string = '' - unsanitized_arg_spec = tf_inspect.getargspec(obj) + unsanitized_arg_spec = tf_inspect.getfullargspec(obj) for clean_attr in ('args', 'varargs', 'keywords'): output_string += '%s=%s, ' % (clean_attr, -- GitLab From 7c2341501a583ca625c976f118090e495cdcbe07 Mon Sep 17 00:00:00 2001 From: Jason Furmanek Date: Wed, 26 Sep 2018 04:44:12 +0000 Subject: [PATCH 0089/1085] Find NCCL2 debians in Tensorflow configure --- configure.py | 136 +++++++++++++++++++--------- third_party/nccl/nccl_configure.bzl | 14 ++- third_party/nccl/system.BUILD.tpl | 4 +- 3 files changed, 105 insertions(+), 49 deletions(-) diff --git a/configure.py b/configure.py index f0b9fada5e..9fd2dc2630 100644 --- a/configure.py +++ b/configure.py @@ -54,6 +54,12 @@ _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) _TF_WORKSPACE = os.path.join(_TF_WORKSPACE_ROOT, 'WORKSPACE') +NCCL_LIB_PATHS = [ + "lib64/", + "lib/powerpc64le-linux-gnu/", + "lib/x86_64-linux-gnu/", + "" +] class UserInputError(Exception): pass @@ -1085,7 +1091,7 @@ def set_tf_tensorrt_install_path(environ_cp): def set_tf_nccl_install_path(environ_cp): - """Set NCCL_INSTALL_PATH and TF_NCCL_VERSION. + """Set NCCL_INSTALL_PATH, NCCL_HDR_PATH and TF_NCCL_VERSION. Args: environ_cp: copy of the os.environ. @@ -1111,46 +1117,98 @@ def set_tf_nccl_install_path(environ_cp): if tf_nccl_version == '1': break # No need to get install path, NCCL 1 is a GitHub repo. - # TODO(csigg): Look with ldconfig first if we can find the library in paths + # Look with ldconfig first if we can find the library in paths # like /usr/lib/x86_64-linux-gnu and the header file in the corresponding # include directory. This is where the NCCL .deb packages install them. - # Then ask the user if we should use that. Instead of a single - # NCCL_INSTALL_PATH, pass separate NCCL_LIB_PATH and NCCL_HDR_PATH to - # nccl_configure.bzl - default_nccl_path = environ_cp.get('CUDA_TOOLKIT_PATH') - ask_nccl_path = (r'Please specify the location where NCCL %s library is ' + + # First check to see if NCCL is in the ldconfig. + # If its found, use that location. + if is_linux(): + ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' + nccl2_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) + nccl2_path_from_ldconfig = re.search('.*libnccl.so .* => (.*)', + nccl2_path_from_ldconfig) + if nccl2_path_from_ldconfig: + nccl2_path_from_ldconfig = nccl2_path_from_ldconfig.group(1) + if os.path.exists('%s.%s' % (nccl2_path_from_ldconfig, tf_nccl_version)): + nccl_install_path = os.path.dirname(nccl2_path_from_ldconfig) + print('NCCL libraries found in ' + nccl2_path_from_ldconfig) + + # Check if this is the main system lib location + if re.search('.*linux-gnu', nccl_install_path): + trunc_nccl_install_path = "/usr" + print("This looks like a system path.") + else: + trunc_nccl_install_path = nccl_install_path + "/.." + + # Look for header + nccl_hdr_path = trunc_nccl_install_path + "/include" + print("Assuming NCCL header path is " + nccl_hdr_path) + if os.path.exists(nccl_hdr_path + "/nccl.h"): + # Set NCCL_INSTALL_PATH + environ_cp['NCCL_INSTALL_PATH'] = nccl_install_path + write_action_env_to_bazelrc('NCCL_INSTALL_PATH', nccl_install_path) + + # Set NCCL_HDR_PATH + environ_cp['NCCL_HDR_PATH'] = nccl_hdr_path + write_action_env_to_bazelrc('NCCL_HDR_PATH', nccl_hdr_path) + break + else: + print('The header for NCCL2 cannot be found. Please install the libnccl-dev package.') + else: + print('NCCL2 is listed by ldconfig but the library is not found. ' + 'Your ldconfig is out of date. Please run sudo ldconfig.') + else: + # NCCL is not found in ldconfig. Ask the user for the location. + default_nccl_path = environ_cp.get('CUDA_TOOLKIT_PATH') + ask_nccl_path = (r'Please specify the location where NCCL %s library is ' 'installed. Refer to README.md for more details. [Default ' 'is %s]:') % (tf_nccl_version, default_nccl_path) - nccl_install_path = get_from_env_or_user_or_default( + nccl_install_path = get_from_env_or_user_or_default( environ_cp, 'NCCL_INSTALL_PATH', ask_nccl_path, default_nccl_path) - # Result returned from "read" will be used unexpanded. That make "~" - # unusable. Going through one more level of expansion to handle that. - nccl_install_path = os.path.realpath(os.path.expanduser(nccl_install_path)) - if is_windows() or is_cygwin(): - nccl_install_path = cygpath(nccl_install_path) - - if is_windows(): - nccl_lib_path = 'lib/x64/nccl.lib' - elif is_linux(): - nccl_lib_path = 'lib/libnccl.so.%s' % tf_nccl_version - elif is_macos(): - nccl_lib_path = 'lib/libnccl.%s.dylib' % tf_nccl_version - - nccl_lib_path = os.path.join(nccl_install_path, nccl_lib_path) - nccl_hdr_path = os.path.join(nccl_install_path, 'include/nccl.h') - if os.path.exists(nccl_lib_path) and os.path.exists(nccl_hdr_path): - # Set NCCL_INSTALL_PATH - environ_cp['NCCL_INSTALL_PATH'] = nccl_install_path - write_action_env_to_bazelrc('NCCL_INSTALL_PATH', nccl_install_path) - break - - # Reset and Retry - print('Invalid path to NCCL %s toolkit, %s or %s not found. Please use the ' - 'O/S agnostic package of NCCL 2' % (tf_nccl_version, nccl_lib_path, + # Result returned from "read" will be used unexpanded. That make "~" + # unusable. Going through one more level of expansion to handle that. + nccl_install_path = os.path.realpath(os.path.expanduser(nccl_install_path)) + if is_windows() or is_cygwin(): + nccl_install_path = cygpath(nccl_install_path) + + if is_windows(): + nccl_lib_path = 'lib/x64/nccl.lib' + elif is_linux(): + nccl_lib_filename = 'libnccl.so.%s' % tf_nccl_version + nccl_lpath = '%s/lib/%s' % (nccl_install_path, nccl_lib_filename) + if not os.path.exists(nccl_lpath): + for relative_path in NCCL_LIB_PATHS: + path = '%s/%s%s' % (nccl_install_path, relative_path, nccl_lib_filename) + if os.path.exists(path): + print("NCCL found at " + path) + nccl_lib_path = path + break + else: + nccl_lib_path = nccl_lpath + elif is_macos(): + nccl_lib_path = 'lib/libnccl.%s.dylib' % tf_nccl_version + + nccl_lib_path = os.path.join(nccl_install_path, nccl_lib_path) + nccl_hdr_path = os.path.join(os.path.dirname(nccl_lib_path), '../include/nccl.h') + print("Assuming NCCL header path is "+nccl_hdr_path) + if os.path.exists(nccl_lib_path) and os.path.exists(nccl_hdr_path): + # Set NCCL_INSTALL_PATH + environ_cp['NCCL_INSTALL_PATH'] = os.path.dirname(nccl_lib_path) + write_action_env_to_bazelrc('NCCL_INSTALL_PATH', os.path.dirname(nccl_lib_path)) + + # Set NCCL_HDR_PATH + environ_cp['NCCL_HDR_PATH'] = os.path.dirname(nccl_hdr_path) + write_action_env_to_bazelrc('NCCL_HDR_PATH', os.path.dirname(nccl_hdr_path)) + break + + # Reset and Retry + print('Invalid path to NCCL %s toolkit, %s or %s not found. Please use the ' + 'O/S agnostic package of NCCL 2' % (tf_nccl_version, nccl_lib_path, nccl_hdr_path)) - environ_cp['TF_NCCL_VERSION'] = '' + environ_cp['TF_NCCL_VERSION'] = '' else: raise UserInputError('Invalid TF_NCCL setting was provided %d ' 'times in a row. Assuming to be a scripting mistake.' % @@ -1401,20 +1459,10 @@ def set_grpc_build_flags(): def set_system_libs_flag(environ_cp): syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') + syslibs = ','.join(sorted(syslibs.split(','))) if syslibs and syslibs != '': - if ',' in syslibs: - syslibs = ','.join(sorted(syslibs.split(','))) - else: - syslibs = ','.join(sorted(syslibs.split())) write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) - if 'PREFIX' in environ_cp: - write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) - if 'LIBDIR' in environ_cp: - write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) - if 'INCLUDEDIR' in environ_cp: - write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) - def set_windows_build_flags(environ_cp): """Set Windows specific build options.""" diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl index ce9447096e..0713b36724 100644 --- a/third_party/nccl/nccl_configure.bzl +++ b/third_party/nccl/nccl_configure.bzl @@ -5,6 +5,7 @@ * `TF_NCCL_VERSION`: The NCCL version. * `NCCL_INSTALL_PATH`: The installation path of the NCCL library. + * `NCCL_HDR_PATH`: The installation path of the NCCL header files. """ load( @@ -15,6 +16,7 @@ load( ) _NCCL_INSTALL_PATH = "NCCL_INSTALL_PATH" +_NCCL_HDR_PATH = "NCCL_HDR_PATH" _TF_NCCL_VERSION = "TF_NCCL_VERSION" _TF_NCCL_CONFIG_REPO = "TF_NCCL_CONFIG_REPO" @@ -68,7 +70,7 @@ def _find_nccl_header(repository_ctx, nccl_install_path): return header_path -def _check_nccl_version(repository_ctx, nccl_install_path, nccl_version): +def _check_nccl_version(repository_ctx, nccl_install_path, nccl_hdr_path, nccl_version): """Checks whether the header file matches the specified version of NCCL. Args: @@ -79,7 +81,9 @@ def _check_nccl_version(repository_ctx, nccl_install_path, nccl_version): Returns: A string containing the library version of NCCL. """ - header_path = _find_nccl_header(repository_ctx, nccl_install_path) + header_path = repository_ctx.path("%s/nccl.h" % nccl_hdr_path) + if not header_path.exists: + header_path = _find_nccl_header(repository_ctx, nccl_install_path) header_dir = str(header_path.realpath.dirname) major_version = find_cuda_define(repository_ctx, header_dir, "nccl.h", _DEFINE_NCCL_MAJOR) @@ -109,6 +113,7 @@ def _find_nccl_lib(repository_ctx, nccl_install_path, nccl_version): """ lib_path = repository_ctx.path("%s/lib/libnccl.so.%s" % (nccl_install_path, nccl_version)) + if not lib_path.exists: auto_configure_fail("Cannot find NCCL library %s" % str(lib_path)) return lib_path @@ -138,10 +143,12 @@ def _nccl_configure_impl(repository_ctx): else: # Create target for locally installed NCCL. nccl_install_path = repository_ctx.os.environ[_NCCL_INSTALL_PATH].strip() - _check_nccl_version(repository_ctx, nccl_install_path, nccl_version) + nccl_hdr_path = repository_ctx.os.environ[_NCCL_HDR_PATH].strip() + _check_nccl_version(repository_ctx, nccl_install_path, nccl_hdr_path, nccl_version) repository_ctx.template("BUILD", _NCCL_LOCAL_BUILD_TEMPLATE, { "%{version}": nccl_version, "%{install_path}": nccl_install_path, + "%{hdr_path}": nccl_hdr_path, }) @@ -149,6 +156,7 @@ nccl_configure = repository_rule( implementation=_nccl_configure_impl, environ=[ _NCCL_INSTALL_PATH, + _NCCL_HDR_PATH, _TF_NCCL_VERSION, ], ) diff --git a/third_party/nccl/system.BUILD.tpl b/third_party/nccl/system.BUILD.tpl index 7ca835dedf..a07f54955f 100644 --- a/third_party/nccl/system.BUILD.tpl +++ b/third_party/nccl/system.BUILD.tpl @@ -20,7 +20,7 @@ genrule( "libnccl.so.%{version}", "nccl.h", ], - cmd = """cp "%{install_path}/include/nccl.h" "$(@D)/nccl.h" && - cp "%{install_path}/lib/libnccl.so.%{version}" "$(@D)/libnccl.so.%{version}" """, + cmd = """cp "%{hdr_path}/nccl.h" "$(@D)/nccl.h" && + cp "%{install_path}/libnccl.so.%{version}" "$(@D)/libnccl.so.%{version}" """, ) -- GitLab From 96eec07af06f4dfc75cee57b74ba4b5347619634 Mon Sep 17 00:00:00 2001 From: Cao Zongyan Date: Wed, 26 Sep 2018 13:04:46 +0800 Subject: [PATCH 0090/1085] Re-add compat module for leaky_relu implementation. --- tensorflow/python/ops/nn_ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 3f64f0af9a..78e000e458 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -22,6 +22,7 @@ import numbers import numpy as np +from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import graph_util -- GitLab From d59678448469ca134875e062f7f8d6d77942af4e Mon Sep 17 00:00:00 2001 From: Jason Furmanek Date: Wed, 26 Sep 2018 05:19:10 +0000 Subject: [PATCH 0091/1085] fix unintential removal of set_system_libs_flag --- configure.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 9fd2dc2630..3791ead3ed 100644 --- a/configure.py +++ b/configure.py @@ -1459,10 +1459,20 @@ def set_grpc_build_flags(): def set_system_libs_flag(environ_cp): syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') - syslibs = ','.join(sorted(syslibs.split(','))) if syslibs and syslibs != '': + if ',' in syslibs: + syslibs = ','.join(sorted(syslibs.split(','))) + else: + syslibs = ','.join(sorted(syslibs.split())) write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) + if 'PREFIX' in environ_cp: + write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) + if 'LIBDIR' in environ_cp: + write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) + if 'INCLUDEDIR' in environ_cp: +write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) + def set_windows_build_flags(environ_cp): """Set Windows specific build options.""" -- GitLab From 1668d28ca3558f3bc4fcf94752799712211f219e Mon Sep 17 00:00:00 2001 From: Jason Furmanek Date: Wed, 26 Sep 2018 05:22:04 +0000 Subject: [PATCH 0092/1085] fix in last line of set_system_lib_flag --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 3791ead3ed..b1ab55b657 100644 --- a/configure.py +++ b/configure.py @@ -1471,7 +1471,7 @@ def set_system_libs_flag(environ_cp): if 'LIBDIR' in environ_cp: write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) if 'INCLUDEDIR' in environ_cp: -write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) + write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) def set_windows_build_flags(environ_cp): -- GitLab From 09bf8eb99cd76c506dcd2a0e8c8e893f7f3916b1 Mon Sep 17 00:00:00 2001 From: Jason Furmanek Date: Wed, 26 Sep 2018 05:26:54 +0000 Subject: [PATCH 0093/1085] white space removal --- third_party/nccl/nccl_configure.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl index 0713b36724..d78fe8f3aa 100644 --- a/third_party/nccl/nccl_configure.bzl +++ b/third_party/nccl/nccl_configure.bzl @@ -113,7 +113,6 @@ def _find_nccl_lib(repository_ctx, nccl_install_path, nccl_version): """ lib_path = repository_ctx.path("%s/lib/libnccl.so.%s" % (nccl_install_path, nccl_version)) - if not lib_path.exists: auto_configure_fail("Cannot find NCCL library %s" % str(lib_path)) return lib_path -- GitLab From d970edea764a5a0937135b7d45061b1d31af6d0a Mon Sep 17 00:00:00 2001 From: Yicheng Fan Date: Wed, 26 Sep 2018 20:18:31 +0800 Subject: [PATCH 0094/1085] Use scatter ops to calculate sparse tensor, which gives 40x speed-up on my job --- tensorflow/python/training/adam.py | 81 ++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 704ad6d3fe..4ace03376d 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -174,7 +175,51 @@ class AdamOptimizer(optimizer.Optimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - def _apply_sparse_shared(self, grad, var, indices, scatter_add): + class ScatterOpWrapper(object): + """Wraps necessary scatter ops for sparse tensors.""" + + def __init__(self, use_locking=False): + self._use_locking = use_locking + + def add(self, sparse_tensor, index, delta): + return state_ops.scatter_add(sparse_tensor, index, delta, + use_locking = self._use_locking) + + def sub(self, sparse_tensor, index, delta): + return state_ops.scatter_sub(sparse_tensor, index, delta, + use_locking = self._use_locking) + + def update(self, sparse_tensor, index, value): + return state_ops.scatter_update(sparse_tensor, index, value, + use_locking = self._use_locking) + + + class ResourceScatterOpWrapper(ScatterOpWrapper): + """Wraps necessay scatter ops for sparse resource variables.""" + + def __init__(self, use_locking=False): + super(AdamOptimizer.ResourceScatterOpWrapper, self).__init__(use_locking) + + def add(self, sparse_tensor, index, delta): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_add( + sparse_tensor.handle, index, delta)]): + return sparse_tensor.value() + + def sub(self, sparse_tensor, index, delta): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_sub( + sparse_tensor.handle, index, delta)]): + return sparse_tensor.value() + + def update(self, sparse_tensor, index, value): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_update( + sparse_tensor.handle, index, value)]): + return sparse_tensor.value() + + + def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) @@ -186,37 +231,29 @@ class AdamOptimizer(optimizer.Optimizer): # m_t = beta1 * m + (1 - beta1) * g_t m = self.get_slot(var, "m") m_scaled_g_values = grad * (1 - beta1_t) - m_t = state_ops.assign(m, m * beta1_t, - use_locking=self._use_locking) - with ops.control_dependencies([m_t]): - m_t = scatter_add(m, indices, m_scaled_g_values) + m_gathered = array_ops.gather(m, indices) + m_t_gathered = m_gathered * beta1_t + m_scaled_g_values + m_t = scatter_op_wrapper.update(m, indices, m_t_gathered) # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) v = self.get_slot(var, "v") v_scaled_g_values = (grad * grad) * (1 - beta2_t) - v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) - with ops.control_dependencies([v_t]): - v_t = scatter_add(v, indices, v_scaled_g_values) - v_sqrt = math_ops.sqrt(v_t) - var_update = state_ops.assign_sub(var, - lr * m_t / (v_sqrt + epsilon_t), - use_locking=self._use_locking) + v_gathered = array_ops.gather(v, indices) + v_t_gathered = v_gathered * beta2_t + v_scaled_g_values + v_t = scatter_op_wrapper.update(v, indices, v_t_gathered) + + v_sqrt_gathered = math_ops.sqrt(v_t_gathered) + var_update = scatter_op_wrapper.sub( + var, indices, lr * m_t_gathered / (v_sqrt_gathered + epsilon_t)) return control_flow_ops.group(*[var_update, m_t, v_t]) + def _apply_sparse(self, grad, var): return self._apply_sparse_shared( - grad.values, var, grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking)) - - def _resource_scatter_add(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add( - x.handle, i, v)]): - return x.value() + grad.values, var, grad.indices, AdamOptimizer.ScatterOpWrapper(self._use_locking)) def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared( - grad, var, indices, self._resource_scatter_add) + grad, var, indices, AdamOptimizer.ResourceScatterOpWrapper(self._use_locking)) def _finish(self, update_ops, name_scope): # Update the power accumulators. -- GitLab From b60c3747594fe7a879378d7b69b60eb9a0c02ca9 Mon Sep 17 00:00:00 2001 From: Yicheng Fan Date: Wed, 26 Sep 2018 20:30:48 +0800 Subject: [PATCH 0095/1085] Fix lint --- tensorflow/python/training/adam.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 4ace03376d..d462dc576f 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -183,15 +183,15 @@ class AdamOptimizer(optimizer.Optimizer): def add(self, sparse_tensor, index, delta): return state_ops.scatter_add(sparse_tensor, index, delta, - use_locking = self._use_locking) + use_locking=self._use_locking) def sub(self, sparse_tensor, index, delta): return state_ops.scatter_sub(sparse_tensor, index, delta, - use_locking = self._use_locking) + use_locking=self._use_locking) def update(self, sparse_tensor, index, value): return state_ops.scatter_update(sparse_tensor, index, value, - use_locking = self._use_locking) + use_locking=self._use_locking) class ResourceScatterOpWrapper(ScatterOpWrapper): @@ -246,14 +246,15 @@ class AdamOptimizer(optimizer.Optimizer): var, indices, lr * m_t_gathered / (v_sqrt_gathered + epsilon_t)) return control_flow_ops.group(*[var_update, m_t, v_t]) - def _apply_sparse(self, grad, var): return self._apply_sparse_shared( - grad.values, var, grad.indices, AdamOptimizer.ScatterOpWrapper(self._use_locking)) + grad.values, var, grad.indices, + AdamOptimizer.ScatterOpWrapper(self._use_locking)) def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared( - grad, var, indices, AdamOptimizer.ResourceScatterOpWrapper(self._use_locking)) + grad, var, indices, + AdamOptimizer.ResourceScatterOpWrapper(self._use_locking)) def _finish(self, update_ops, name_scope): # Update the power accumulators. -- GitLab From fa76895ad577246a8ab241e668765cad651558fb Mon Sep 17 00:00:00 2001 From: Isaac Burbank Date: Wed, 26 Sep 2018 11:20:44 -0600 Subject: [PATCH 0096/1085] Update python_object_to_proto_visitor.py Changed test key for FullArgSpec to check for `varkw`, replacing the old ArgSpec key `keywords` --- tensorflow/tools/api/lib/python_object_to_proto_visitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py index 2a40caf720..a8e69fda4f 100644 --- a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py +++ b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py @@ -49,7 +49,7 @@ def _SanitizedArgSpec(obj): output_string = '' unsanitized_arg_spec = tf_inspect.getfullargspec(obj) - for clean_attr in ('args', 'varargs', 'keywords'): + for clean_attr in ('args', 'varargs', 'varkw'): output_string += '%s=%s, ' % (clean_attr, getattr(unsanitized_arg_spec, clean_attr)) -- GitLab From 5bbcdb8a58efd97b0f73927218d5896da67f5203 Mon Sep 17 00:00:00 2001 From: Isaac Burbank Date: Wed, 26 Sep 2018 11:34:38 -0600 Subject: [PATCH 0097/1085] Update tf_inspect_test.py Remove subsection of added tests that were problematic. --- tensorflow/python/util/tf_inspect_test.py | 78 ----------------------- 1 file changed, 78 deletions(-) diff --git a/tensorflow/python/util/tf_inspect_test.py b/tensorflow/python/util/tf_inspect_test.py index 55f88f8fc6..ba9430c756 100644 --- a/tensorflow/python/util/tf_inspect_test.py +++ b/tensorflow/python/util/tf_inspect_test.py @@ -353,41 +353,6 @@ class TfInspectTest(test.TestCase): self.assertEqual(argspec, tf_inspect.getfullargspec(partial_function)) - def testGetFullArgSpecOnPartialInvalidFullArgSpec(self): - """Tests getfullargspec. - - Tests on partial function that doesn't have valid fullargspec. - """ - - def func(m, n, l, k=4): - return 2 * m + l + n * k - - partial_func = functools.partial(func, n=7) - - exception_message = (r"Some arguments \['l'\] do not have default value, " - "but they are positioned after those with default " - "values. This can not be expressed with ArgSpec.") - with self.assertRaisesRegexp(ValueError, exception_message): - tf_inspect.getfullargspec(partial_func) - - def testGetFullArgSpecOnPartialValidFullArgSpec(self): - """Tests getfullargspec on partial function with valid fullargspec.""" - - def func(m, n, l, k=4): - return 2 * m + l + n * k - - partial_func = functools.partial(func, n=7, l=2) - argspec = tf_inspect.FullArgSpec( - args=['m', 'n', 'l', 'k'], - varargs=None, - varkw=None, - defaults=(7, 2, 4), - kwonlyargs=[], - kwonlydefaults=None, - annotations={}) - - self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - def testGetFullArgSpecOnPartialNoArgumentsLeft(self): """Tests getfullargspec on partial function that prunes all arguments.""" @@ -401,35 +366,6 @@ class TfInspectTest(test.TestCase): self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - def testGetFullArgSpecOnPartialKeywordArgument(self): - """Tests getfullargspec on partial function that prunes some arguments.""" - - def func(m, n): - return 2 * m + n - - partial_func = functools.partial(func, n=7) - argspec = tf_inspect.FullArgSpec( - args=['m', 'n'], varargs=None, varkw=None, defaults=(7,), - kwonlyargs=[], kwonlydefaults=None, annotations={}) - - self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - - def testGetFullArgSpecOnPartialKeywordArgumentWithDefaultValue(self): - """Tests getfullargspec. - - Tests on partial function that prunes argument by keyword. - """ - - def func(m=1, n=2): - return 2 * m + n - - partial_func = functools.partial(func, n=7) - argspec = tf_inspect.FullArgSpec( - args=['m', 'n'], varargs=None, varkw=None, defaults=(1, 7), - kwonlyargs=[], kwonlydefaults=None, annotations={}) - - self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - def testGetFullArgSpecOnPartialWithVarargs(self): """Tests getfullargspec on partial function with variable arguments.""" @@ -459,20 +395,6 @@ class TfInspectTest(test.TestCase): self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - def testGetFullArgSpecOnPartialWithDecorator(self): - """Tests getfullargspec on decorated partial function.""" - - @test_decorator('decorator') - def func(m=1, n=2): - return 2 * m + n - - partial_func = functools.partial(func, n=7) - argspec = tf_inspect.FullArgSpec( - args=['m', 'n'], varargs=None, varkw=None, defaults=(1, 7), - kwonlyargs=[], kwonlydefaults=None, annotations={}) - - self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func)) - def testGetFullArgSpecOnCallableObject(self): class Callable(object): -- GitLab From 3c01644ed3fad31ac1b09afe31e655bd8892f02b Mon Sep 17 00:00:00 2001 From: William Irons Date: Wed, 26 Sep 2018 14:36:45 -0400 Subject: [PATCH 0098/1085] Artifact links for ppc64le GPU builds. whl files for tensorflow_gpu..._ppc6le are now hosted on the OSU Jenkins build server. Nightly builds and Stable Release builds are provided. I didn't include the version number so we won't need to update the readme file for every new release --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57efb876c9..4f57aca99f 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,8 @@ The TensorFlow project strives to abide by generally accepted best practices in | --- | --- | --- | | **IBM s390x** | [![Build Status](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/badge/icon)](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/) | TBA | | **IBM ppc64le CPU** | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/) | TBA | -| **IBM ppc64le GPU** | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_PPC64LE_GPU/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_PPC64LE_GPU/) | TBA | +| **IBM ppc64le GPU** Nightly | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/) | [Nightly](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/) | +| **IBM ppc64le GPU** Stable Release | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) | [Release](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) | | **Linux CPU with Intel® MKL-DNN** Nightly | [![Build Status](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/badge/icon)](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/) | [Nightly](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-whl-nightly/) | | **Linux CPU with Intel® MKL-DNN** Python 2.7
**Linux CPU with Intel® MKL-DNN** Python 3.5
**Linux CPU with Intel® MKL-DNN** Python 3.6 | [![Build Status](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-release-whl/badge/icon)](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-release-whl/lastStableBuild)|[1.10.0 py2.7](https://storage.googleapis.com/intel-optimized-tensorflow/tensorflow-1.10.0-cp27-cp27mu-linux_x86_64.whl)
[1.10.0 py3.5](https://storage.googleapis.com/intel-optimized-tensorflow/tensorflow-1.10.0-cp35-cp35m-linux_x86_64.whl)
[1.10.0 py3.6](https://storage.googleapis.com/intel-optimized-tensorflow/tensorflow-1.10.0-cp36-cp36m-linux_x86_64.whl) | -- GitLab From df26537ea43493a086f279a5ae8262dc79d03f20 Mon Sep 17 00:00:00 2001 From: Yicheng Fan Date: Thu, 27 Sep 2018 11:05:11 +0800 Subject: [PATCH 0099/1085] Fix tests failure in contrib, and hide inner class from public API --- .../contrib/opt/python/training/adamax.py | 22 +++++-------------- .../opt/python/training/nadam_optimizer.py | 6 ++--- tensorflow/python/training/adam.py | 12 +++++----- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 686bac0d84..9c954ca298 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -134,8 +134,7 @@ class AdaMaxOptimizer(adam.AdamOptimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - def _apply_sparse_shared(self, grad, var, indices, - scatter_add, scatter_update): + def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): beta1_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) @@ -147,38 +146,29 @@ class AdaMaxOptimizer(adam.AdamOptimizer): m_slice = array_ops.gather(m, indices) m_t_slice = m_slice * beta1_t + grad * (1 - beta1_t) with ops.control_dependencies([m_t_slice]): - m_t = scatter_update(m, indices, m_t_slice) + m_t = scatter_op_wrapper.update(m, indices, m_t_slice) # u_t = max(beta2 * u, abs(g_t)) v = self.get_slot(var, "v") v_slice = array_ops.gather(v, indices) v_t_slice = math_ops.maximum(v_slice * beta2_t, math_ops.abs(grad)) with ops.control_dependencies([v_t_slice]): - v_t = scatter_update(v, indices, v_t_slice) + v_t = scatter_op_wrapper.update(v, indices, v_t_slice) # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t var_slice = -lr_t / (1 - beta1_power) * (m_t_slice / (v_t_slice + epsilon_t)) with ops.control_dependencies([var_slice]): - var_update = scatter_add(var, indices, var_slice) + var_update = scatter_op_wrapper.add(var, indices, var_slice) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): return self._apply_sparse_shared( grad.values, var, grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking), - lambda x, i, v: state_ops.scatter_update( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking)) - - def _resource_scatter_update(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_update( - x.handle, i, v)]): - return x.value() + adam.AdamOptimizer._ScatterOpWrapper(self._use_locking)) def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared( grad, var, indices, - self._resource_scatter_add, self._resource_scatter_update) + adam.AdamOptimizer._ResourceScatterOpWrapper()) def _finish(self, update_ops, name_scope): # Update the power accumulators. diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer.py b/tensorflow/contrib/opt/python/training/nadam_optimizer.py index 44a8890cb1..fc08d5827d 100644 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer.py +++ b/tensorflow/contrib/opt/python/training/nadam_optimizer.py @@ -67,7 +67,7 @@ class NadamOptimizer(adam.AdamOptimizer): use_locking=self._use_locking, use_nesterov=True) - def _apply_sparse_shared(self, grad, var, indices, scatter_add): + def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) @@ -81,7 +81,7 @@ class NadamOptimizer(adam.AdamOptimizer): m_scaled_g_values = grad * (1 - beta1_t) m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) with ops.control_dependencies([m_t]): - m_t = scatter_add(m, indices, m_scaled_g_values) + m_t = scatter_op_wrapper.add(m, indices, m_scaled_g_values) # m_bar = (1 - beta1) * g_t + beta1 * m_t m_bar = m_scaled_g_values + beta1_t * m_t # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) @@ -89,7 +89,7 @@ class NadamOptimizer(adam.AdamOptimizer): v_scaled_g_values = (grad * grad) * (1 - beta2_t) v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) with ops.control_dependencies([v_t]): - v_t = scatter_add(v, indices, v_scaled_g_values) + v_t = scatter_op_wrapper.add(v, indices, v_scaled_g_values) v_sqrt = math_ops.sqrt(v_t) var_update = state_ops.assign_sub( var, lr * m_bar / (v_sqrt + epsilon_t), use_locking=self._use_locking) diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index d462dc576f..05ab859274 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -175,7 +175,7 @@ class AdamOptimizer(optimizer.Optimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - class ScatterOpWrapper(object): + class _ScatterOpWrapper(object): """Wraps necessary scatter ops for sparse tensors.""" def __init__(self, use_locking=False): @@ -194,11 +194,11 @@ class AdamOptimizer(optimizer.Optimizer): use_locking=self._use_locking) - class ResourceScatterOpWrapper(ScatterOpWrapper): + class _ResourceScatterOpWrapper(_ScatterOpWrapper): """Wraps necessay scatter ops for sparse resource variables.""" - def __init__(self, use_locking=False): - super(AdamOptimizer.ResourceScatterOpWrapper, self).__init__(use_locking) + def __init__(self): + pass def add(self, sparse_tensor, index, delta): with ops.control_dependencies( @@ -249,12 +249,12 @@ class AdamOptimizer(optimizer.Optimizer): def _apply_sparse(self, grad, var): return self._apply_sparse_shared( grad.values, var, grad.indices, - AdamOptimizer.ScatterOpWrapper(self._use_locking)) + AdamOptimizer._ScatterOpWrapper(self._use_locking)) def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared( grad, var, indices, - AdamOptimizer.ResourceScatterOpWrapper(self._use_locking)) + AdamOptimizer._ResourceScatterOpWrapper()) def _finish(self, update_ops, name_scope): # Update the power accumulators. -- GitLab From 4c0c4bb3fe0d68833cf7888e1c164b20d9bfcea0 Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 27 Sep 2018 16:52:51 +0200 Subject: [PATCH 0100/1085] Added chief to the default device_filters for /job:ps This behaviour matches the description in the _get_default_session_config_distributed docstring, and restores the symmetry of the default device_filters. --- tensorflow/python/estimator/run_config.py | 2 +- tensorflow/python/estimator/run_config_test.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index 3773810a04..1995f50733 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -566,7 +566,7 @@ class RunConfig(object): elif self._task_type == TaskType.WORKER: device_filters = ['/job:ps', '/job:worker/task:%d' % self._task_id] elif self._task_type == TaskType.PS: - device_filters = ['/job:ps', '/job:worker', '/job:master'] + device_filters = ['/job:ps', '/job:worker', '/job:chief', '/job:master'] else: # If the task_type is `EVALUATOR` or something other than the ones in # TaskType then don't set any device filters. diff --git a/tensorflow/python/estimator/run_config_test.py b/tensorflow/python/estimator/run_config_test.py index 06df7cb9dd..313bf62c05 100644 --- a/tensorflow/python/estimator/run_config_test.py +++ b/tensorflow/python/estimator/run_config_test.py @@ -1196,8 +1196,9 @@ class RunConfigSessionConfigTest(test.TestCase): } } run_config = _create_run_config_with_cluster_spec(tf_config) - self._assert_equal_session_config(run_config.session_config, - ['/job:ps', '/job:worker', '/job:master']) + self._assert_equal_session_config( + run_config.session_config, + ['/job:ps', '/job:worker', '/job:chief', '/job:master']) def test_evaluator_session_config(self): tf_config = { -- GitLab From 33023751e01b90fcc461dcf8deb41505fa1b62b6 Mon Sep 17 00:00:00 2001 From: Yicheng Fan Date: Fri, 28 Sep 2018 11:19:24 +0800 Subject: [PATCH 0101/1085] use variable.scatter_add/sub/update --- .../contrib/opt/python/training/adamax.py | 19 +++--- .../opt/python/training/nadam_optimizer.py | 8 ++- tensorflow/python/training/adam.py | 62 +++---------------- 3 files changed, 21 insertions(+), 68 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 9c954ca298..debb6d99af 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -134,7 +134,7 @@ class AdaMaxOptimizer(adam.AdamOptimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): + def _apply_sparse_shared(self, grad, var, indices): beta1_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) @@ -146,29 +146,28 @@ class AdaMaxOptimizer(adam.AdamOptimizer): m_slice = array_ops.gather(m, indices) m_t_slice = m_slice * beta1_t + grad * (1 - beta1_t) with ops.control_dependencies([m_t_slice]): - m_t = scatter_op_wrapper.update(m, indices, m_t_slice) + m_t = m.scatter_update(ops.IndexedSlices(m_t_slice, indices), + use_locking=self._use_locking) # u_t = max(beta2 * u, abs(g_t)) v = self.get_slot(var, "v") v_slice = array_ops.gather(v, indices) v_t_slice = math_ops.maximum(v_slice * beta2_t, math_ops.abs(grad)) with ops.control_dependencies([v_t_slice]): - v_t = scatter_op_wrapper.update(v, indices, v_t_slice) + v_t = v.scatter_update(ops.IndexedSlices(v_t_slice, indices), + use_locking=self._use_locking) # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t var_slice = -lr_t / (1 - beta1_power) * (m_t_slice / (v_t_slice + epsilon_t)) with ops.control_dependencies([var_slice]): - var_update = scatter_op_wrapper.add(var, indices, var_slice) + var_update = var.scatter_add(ops.IndexedSlices(var_slice, indices), + use_locking=self._use_locking) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): - return self._apply_sparse_shared( - grad.values, var, grad.indices, - adam.AdamOptimizer._ScatterOpWrapper(self._use_locking)) + return self._apply_sparse_shared(grad.values, var, grad.indices) def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared( - grad, var, indices, - adam.AdamOptimizer._ResourceScatterOpWrapper()) + return self._apply_sparse_shared(grad, var, indices) def _finish(self, update_ops, name_scope): # Update the power accumulators. diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer.py b/tensorflow/contrib/opt/python/training/nadam_optimizer.py index fc08d5827d..208909d2da 100644 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer.py +++ b/tensorflow/contrib/opt/python/training/nadam_optimizer.py @@ -67,7 +67,7 @@ class NadamOptimizer(adam.AdamOptimizer): use_locking=self._use_locking, use_nesterov=True) - def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): + def _apply_sparse_shared(self, grad, var, indices): beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) @@ -81,7 +81,8 @@ class NadamOptimizer(adam.AdamOptimizer): m_scaled_g_values = grad * (1 - beta1_t) m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) with ops.control_dependencies([m_t]): - m_t = scatter_op_wrapper.add(m, indices, m_scaled_g_values) + m_t = m.scatter_add(ops.IndexedSlices(m_scaled_g_values, indices), + use_locking=self._use_locking) # m_bar = (1 - beta1) * g_t + beta1 * m_t m_bar = m_scaled_g_values + beta1_t * m_t # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) @@ -89,7 +90,8 @@ class NadamOptimizer(adam.AdamOptimizer): v_scaled_g_values = (grad * grad) * (1 - beta2_t) v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) with ops.control_dependencies([v_t]): - v_t = scatter_op_wrapper.add(v, indices, v_scaled_g_values) + v_t = v.scatter_add(ops.IndexedSlices(v_scaled_g_values, indices), + use_locking=self._use_locking) v_sqrt = math_ops.sqrt(v_t) var_update = state_ops.assign_sub( var, lr * m_bar / (v_sqrt + epsilon_t), use_locking=self._use_locking) diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 05ab859274..4b31fac6c3 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -175,51 +175,7 @@ class AdamOptimizer(optimizer.Optimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - class _ScatterOpWrapper(object): - """Wraps necessary scatter ops for sparse tensors.""" - - def __init__(self, use_locking=False): - self._use_locking = use_locking - - def add(self, sparse_tensor, index, delta): - return state_ops.scatter_add(sparse_tensor, index, delta, - use_locking=self._use_locking) - - def sub(self, sparse_tensor, index, delta): - return state_ops.scatter_sub(sparse_tensor, index, delta, - use_locking=self._use_locking) - - def update(self, sparse_tensor, index, value): - return state_ops.scatter_update(sparse_tensor, index, value, - use_locking=self._use_locking) - - - class _ResourceScatterOpWrapper(_ScatterOpWrapper): - """Wraps necessay scatter ops for sparse resource variables.""" - - def __init__(self): - pass - - def add(self, sparse_tensor, index, delta): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add( - sparse_tensor.handle, index, delta)]): - return sparse_tensor.value() - - def sub(self, sparse_tensor, index, delta): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_sub( - sparse_tensor.handle, index, delta)]): - return sparse_tensor.value() - - def update(self, sparse_tensor, index, value): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_update( - sparse_tensor.handle, index, value)]): - return sparse_tensor.value() - - - def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper): + def _apply_sparse_shared(self, grad, var, indices): beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) @@ -233,28 +189,24 @@ class AdamOptimizer(optimizer.Optimizer): m_scaled_g_values = grad * (1 - beta1_t) m_gathered = array_ops.gather(m, indices) m_t_gathered = m_gathered * beta1_t + m_scaled_g_values - m_t = scatter_op_wrapper.update(m, indices, m_t_gathered) + m_t = m.scatter_update(ops.IndexedSlices(m_t_gathered, indices)) # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) v = self.get_slot(var, "v") v_scaled_g_values = (grad * grad) * (1 - beta2_t) v_gathered = array_ops.gather(v, indices) v_t_gathered = v_gathered * beta2_t + v_scaled_g_values - v_t = scatter_op_wrapper.update(v, indices, v_t_gathered) + v_t = v.scatter_update(ops.IndexedSlices(v_t_gathered, indices)) v_sqrt_gathered = math_ops.sqrt(v_t_gathered) - var_update = scatter_op_wrapper.sub( - var, indices, lr * m_t_gathered / (v_sqrt_gathered + epsilon_t)) + var_update = var.scatter_sub(ops.IndexedSlices( + lr * m_t_gathered / (v_sqrt_gathered + epsilon_t), indices)) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): - return self._apply_sparse_shared( - grad.values, var, grad.indices, - AdamOptimizer._ScatterOpWrapper(self._use_locking)) + return self._apply_sparse_shared(grad.values, var, grad.indices) def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared( - grad, var, indices, - AdamOptimizer._ResourceScatterOpWrapper()) + return self._apply_sparse_shared(grad, var, indices) def _finish(self, update_ops, name_scope): # Update the power accumulators. -- GitLab From efe17306442aa91192df953ae537d3f9b824dae6 Mon Sep 17 00:00:00 2001 From: IMBurbank Date: Thu, 27 Sep 2018 22:21:47 -0600 Subject: [PATCH 0102/1085] Updated python3 tf_inspect.getargspec calls to use getfullargspec and repackage the return values into the getargspec struct. --- .../python/losses/python/tuple_losses_impl.py | 2 +- .../labeled_tensor/python/ops/_typecheck.py | 2 +- .../layers/python/layers/rev_block_lib.py | 3 +- .../python/learn/estimators/estimator.py | 4 +- .../learn/python/learn/estimators/head.py | 2 +- .../learn/python/learn/experiment_test.py | 2 +- .../learn/python/learn/export_strategy.py | 2 +- .../contrib/learn/python/learn/metric_spec.py | 2 +- .../contrib/learn/python/learn/monitors.py | 2 +- .../contrib/tpu/python/tpu/tpu_function.py | 2 +- tensorflow/python/framework/errors_impl.py | 2 +- tensorflow/python/framework/function.py | 6 +- tensorflow/python/keras/backend_test.py | 2 +- tensorflow/python/keras/testing_utils.py | 2 +- .../kernel_tests/variable_scope_test.py | 4 +- tensorflow/python/ops/variable_scope.py | 4 +- tensorflow/python/util/tf_contextlib_test.py | 2 +- tensorflow/python/util/tf_inspect.py | 89 ++++++++++++------- .../api/lib/python_object_to_proto_visitor.py | 4 +- 19 files changed, 79 insertions(+), 59 deletions(-) diff --git a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py index 00a83e5e55..221c70c38b 100644 --- a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py +++ b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py @@ -101,7 +101,7 @@ def _args_to_gan_model(loss_fn): """ # Match arguments in `loss_fn` to elements of `namedtuple`. # TODO(joelshor): Properly handle `varargs` and `keywords`. - argspec = tf_inspect.getfullargspec(loss_fn) + argspec = tf_inspect.getargspec(loss_fn) defaults = argspec.defaults or [] required_args = set(argspec.args[:-len(defaults)]) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py b/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py index 0e23039847..80fa17ec1f 100644 --- a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py +++ b/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py @@ -230,7 +230,7 @@ def accepts(*types): def check_accepts(f): """Check the types.""" - spec = tf_inspect.getfullargspec(f) + spec = tf_inspect.getargspec(f) num_function_arguments = len(spec.args) if len(types) != num_function_arguments: diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py index 55979cc391..06da32072f 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py @@ -576,8 +576,7 @@ def _recomputing_grad_fn(compute_fn, def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): """See recompute_grad.""" - has_is_recompute_kwarg = ( - "is_recomputing" in tf_inspect.getfullargspec(fn).args) + has_is_recompute_kwarg = "is_recomputing" in tf_inspect.getargspec(fn).args for arg in args: if not isinstance(arg, framework_ops.Tensor): raise ValueError("All inputs to function must be Tensors") diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index b88923bca2..c1de42782e 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -199,11 +199,11 @@ def _model_fn_args(fn): if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): # Handle functools.partial and similar objects. return tuple([ - arg for arg in tf_inspect.getfullargspec(fn.func).args[len(fn.args):] + arg for arg in tf_inspect.getargspec(fn.func).args[len(fn.args):] if arg not in set(fn.keywords.keys()) ]) # Handle function. - return tuple(tf_inspect.getfullargspec(fn).args) + return tuple(tf_inspect.getargspec(fn).args) def _get_replica_device_setter(config): diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index 63dd08316b..c6f79e00d5 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -1861,7 +1861,7 @@ def _get_arguments(func): _, func = tf_decorator.unwrap(func) if hasattr(func, "__code__"): # Regular function. - return tf_inspect.getfullargspec(func) + return tf_inspect.getargspec(func) elif hasattr(func, "func"): # Partial function. return _get_arguments(func.func) diff --git a/tensorflow/contrib/learn/python/learn/experiment_test.py b/tensorflow/contrib/learn/python/learn/experiment_test.py index 6926696fb6..fb16c94c29 100644 --- a/tensorflow/contrib/learn/python/learn/experiment_test.py +++ b/tensorflow/contrib/learn/python/learn/experiment_test.py @@ -126,7 +126,7 @@ class TestBaseEstimator(object): def _check_method_supports_args(method, kwargs): """Checks that the given method supports the given args.""" - supported_args = tuple(tf_inspect.getfullargspec(method).args) + supported_args = tuple(tf_inspect.getargspec(method).args) for kwarg in kwargs: if kwarg not in supported_args: raise ValueError( diff --git a/tensorflow/contrib/learn/python/learn/export_strategy.py b/tensorflow/contrib/learn/python/learn/export_strategy.py index 0d6e0cdc18..075cab536e 100644 --- a/tensorflow/contrib/learn/python/learn/export_strategy.py +++ b/tensorflow/contrib/learn/python/learn/export_strategy.py @@ -96,7 +96,7 @@ class ExportStrategy( """ # don't break existing export_fns that don't accept checkpoint_path and # eval_result - export_fn_args = tf_inspect.getfullargspec(self.export_fn).args + export_fn_args = tf_inspect.getargspec(self.export_fn).args kwargs = {} if 'checkpoint_path' in export_fn_args: kwargs['checkpoint_path'] = checkpoint_path diff --git a/tensorflow/contrib/learn/python/learn/metric_spec.py b/tensorflow/contrib/learn/python/learn/metric_spec.py index 604d6d46b4..97220365d5 100644 --- a/tensorflow/contrib/learn/python/learn/metric_spec.py +++ b/tensorflow/contrib/learn/python/learn/metric_spec.py @@ -51,7 +51,7 @@ def _args(fn): return tuple( [arg for arg in _args(fn.func) if arg not in set(fn.keywords.keys())]) # Handle function. - return tuple(tf_inspect.getfullargspec(fn).args) + return tuple(tf_inspect.getargspec(fn).args) _CANONICAL_LABELS_ARG = 'labels' diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py index 5f61e0264f..3d691d4340 100644 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ b/tensorflow/contrib/learn/python/learn/monitors.py @@ -1303,7 +1303,7 @@ class RunHookAdapterForMonitors(session_run_hook.SessionRunHook): def end(self, session): self._last_step = None for m in self._monitors: - if "session" in tf_inspect.getfullargspec(m.end).args: + if "session" in tf_inspect.getargspec(m.end).args: m.end(session=session) else: m.end() diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_function.py b/tensorflow/contrib/tpu/python/tpu/tpu_function.py index 9c4bd1c4d1..0c7a38dbbb 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_function.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_function.py @@ -80,7 +80,7 @@ def check_function_argument_count(func, input_arity, infeed_queue): number_of_arguments_needed = input_arity if infeed_queue is not None: number_of_arguments_needed += infeed_queue.number_of_tuple_elements - arg_spec = tf_inspect.getfullargspec(func) + arg_spec = tf_inspect.getargspec(func) number_of_args = len(arg_spec.args) if arg_spec.defaults is None: number_of_defaults = 0 diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py index c373e75a74..5af71f2cfb 100644 --- a/tensorflow/python/framework/errors_impl.py +++ b/tensorflow/python/framework/errors_impl.py @@ -55,7 +55,7 @@ class OpError(Exception): def __reduce__(self): # Allow the subclasses to accept less arguments in their __init__. - init_argspec = tf_inspect.getfullargspec(self.__class__.__init__) + init_argspec = tf_inspect.getargspec(self.__class__.__init__) args = tuple(getattr(self, arg) for arg in init_argspec.args[1:]) return self.__class__, args diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index 3db6f683c9..225208944e 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -132,9 +132,9 @@ class Defun(object): raise ValueError("func %s must be callable" % func) # Func should not use kwargs and defaults. - argspec = tf_inspect.getfullargspec(func) - if argspec.varkw or argspec.defaults: - raise ValueError("Functions with argument defaults or varkw " + argspec = tf_inspect.getargspec(func) + if argspec.keywords or argspec.defaults: + raise ValueError("Functions with argument defaults or keywords " "arguments are not supported.") # Computes how many arguments 'func' has. diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py index 31191d0d35..ab71589940 100644 --- a/tensorflow/python/keras/backend_test.py +++ b/tensorflow/python/keras/backend_test.py @@ -452,7 +452,7 @@ class BackendLinearAlgebraTest(test.TestCase): compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5), keras_kwargs={'axis': -1}, np_kwargs={'axis': -1}) - if 'keepdims' in tf_inspect.getfullargspec(keras_op).args: + if 'keepdims' in tf_inspect.getargspec(keras_op).args: compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5), keras_kwargs={'axis': 1, diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py index 1afaba5653..501b50ba5f 100644 --- a/tensorflow/python/keras/testing_utils.py +++ b/tensorflow/python/keras/testing_utils.py @@ -102,7 +102,7 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None, layer.set_weights(weights) # test and instantiation from weights - if 'weights' in tf_inspect.getfullargspec(layer_cls.__init__): + if 'weights' in tf_inspect.getargspec(layer_cls.__init__): kwargs['weights'] = weights layer = layer_cls(**kwargs) diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index 1d0b72b17a..401e1ae102 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -998,8 +998,8 @@ class VariableScopeTest(test.TestCase): def testSignatureGetVarVsGetLocalVar(self): """get_{local,}variable() must take the same list of args.""" - arg_names = tf_inspect.getfullargspec(variable_scope.get_variable)[0] - local_arg_names = tf_inspect.getfullargspec( + arg_names = tf_inspect.getargspec(variable_scope.get_variable)[0] + local_arg_names = tf_inspect.getargspec( variable_scope.get_local_variable)[0] self.assertEqual(arg_names, local_arg_names) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 3cc1eb916d..a43676cd70 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -892,14 +892,14 @@ class _VariableStore(object): if shape and shape.is_fully_defined(): init_val = lambda: initializer( # pylint: disable=g-long-lambda shape.as_list(), dtype=dtype, partition_info=partition_info) - elif not tf_inspect.getfullargspec(initializer).args: + elif not tf_inspect.getargspec(initializer).args: init_val = initializer else: raise ValueError("You can only pass an initializer function that " "expects no arguments to its callable when the " "shape is not fully defined. The given initializer " "function expects the following args %s" % - tf_inspect.getfullargspec(initializer).args) + tf_inspect.getargspec(initializer).args) variable_dtype = dtype.base_dtype # Create the variable. diff --git a/tensorflow/python/util/tf_contextlib_test.py b/tensorflow/python/util/tf_contextlib_test.py index 1e921b5ea3..4a5bf388a6 100644 --- a/tensorflow/python/util/tf_contextlib_test.py +++ b/tensorflow/python/util/tf_contextlib_test.py @@ -83,7 +83,7 @@ class TfContextlibTest(test.TestCase): self.assertFalse(isinstance(target, tf_decorator.TFDecorator)) def testGetArgSpecReturnsWrappedArgSpec(self): - argspec = tf_inspect.getfullargspec(test_params_and_defaults) + argspec = tf_inspect.getargspec(test_params_and_defaults) self.assertEqual(['a', 'b', 'c', 'd'], argspec.args) self.assertEqual((2, True, 'hello'), argspec.defaults) diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py index 234850ac3f..3cd6c515b9 100644 --- a/tensorflow/python/util/tf_inspect.py +++ b/tensorflow/python/util/tf_inspect.py @@ -36,6 +36,53 @@ else: 'annotations' ]) +if hasattr(_inspect, 'getfullargspec'): + _getfullargspec = _inspect.getfullargspec # pylint: disable=invalid-name + + def _getargspec(target): + """A python3 version of getargspec. + + Calls `getfullargspec` and assigns args, varargs, + varkw, and defaults to a python 2/3 compatible `ArgSpec`. + + The parameter name 'varkw' is changed to 'keywords' to fit the + `ArgSpec` struct. + + Args: + target: the target object to inspect. + Returns: + An ArgSpec with args, varargs, keywords, and defaults parameters + from FullArgSpec. + """ + fullargspecs = getfullargspec(target) + argspecs = ArgSpec( + args=fullargspecs.args, + varargs=fullargspecs.varargs, + keywords=fullargspecs.varkw, + defaults=fullargspecs.defaults) + return argspecs +else: + _getargspec = _inspect.getargspec + + def _getfullargspec(target): + """A python2 version of getfullargspec. + + Args: + target: the target object to inspect. + Returns: + A FullArgSpec with empty kwonlyargs, kwonlydefaults and annotations. + """ + argspecs = getargspec(target) + fullargspecs = FullArgSpec( + args=argspecs.args, + varargs=argspecs.varargs, + varkw=argspecs.keywords, + defaults=argspecs.defaults, + kwonlyargs=[], + kwonlydefaults=None, + annotations={}) + return fullargspecs + def currentframe(): """TFDecorator-aware replacement for inspect.currentframe.""" @@ -45,10 +92,8 @@ def currentframe(): def getargspec(obj): """TFDecorator-aware replacement for `inspect.getargspec`. - This should not be called from other modules. It is deprecated in python3. - - Use `getfullargspec`. It is a TFDecorator-aware replacement for - `inspect.getfullargspec` compatible with both python2 and python3. + Note: `getfullargspec` is recommended as the python 2/3 compatible + replacement for this function. Args: obj: A function, partial function, or callable object, possibly @@ -56,8 +101,8 @@ def getargspec(obj): Returns: The `ArgSpec` that describes the signature of the outermost decorator that - changes the callable's signature. If the callable is not decorated, - `inspect.getargspec()` will be called directly on the object. + changes the callable's signature, or the `ArgSpec` that describes + the object if not decorated. Raises: ValueError: When callable's signature can not be expressed with @@ -77,24 +122,24 @@ def getargspec(obj): try: # Python3 will handle most callables here (not partial). - return _inspect.getargspec(target) + return _getargspec(target) except TypeError: pass if isinstance(target, type): try: - return _inspect.getargspec(target.__init__) + return _getargspec(target.__init__) except TypeError: pass try: - return _inspect.getargspec(target.__new__) + return _getargspec(target.__new__) except TypeError: pass # The `type(target)` ensures that if a class is received we don't return # the signature of it's __call__ method. - return _inspect.getargspec(type(target).__call__) + return _getargspec(type(target).__call__) def _get_argspec_for_partial(obj): @@ -177,30 +222,6 @@ def _get_argspec_for_partial(obj): return ArgSpec(args, varargs, keywords, tuple(all_defaults[first_default:])) -if hasattr(_inspect, 'getfullargspec'): - _getfullargspec = _inspect.getfullargspec -else: - - def _getfullargspec(target): - """A python2 version of getfullargspec. - - Args: - target: the target object to inspect. - Returns: - A FullArgSpec with empty kwonlyargs, kwonlydefaults and annotations. - """ - argspecs = getargspec(target) - fullargspecs = FullArgSpec( - args=argspecs.args, - varargs=argspecs.varargs, - varkw=argspecs.keywords, - defaults=argspecs.defaults, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}) - return fullargspecs - - def getfullargspec(obj): """TFDecorator-aware replacement for `inspect.getfullargspec`. diff --git a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py index a8e69fda4f..3a48cf683c 100644 --- a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py +++ b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py @@ -47,9 +47,9 @@ def _SanitizedArgSpec(obj): string, a string representation of the argspec. """ output_string = '' - unsanitized_arg_spec = tf_inspect.getfullargspec(obj) + unsanitized_arg_spec = tf_inspect.getargspec(obj) - for clean_attr in ('args', 'varargs', 'varkw'): + for clean_attr in ('args', 'varargs', 'keywords'): output_string += '%s=%s, ' % (clean_attr, getattr(unsanitized_arg_spec, clean_attr)) -- GitLab From 307a095da517a7382f66e14273464c85296425aa Mon Sep 17 00:00:00 2001 From: Rin Arakaki Date: Fri, 28 Sep 2018 04:59:16 +0000 Subject: [PATCH 0103/1085] Modify docs to conform to Python syntax --- tensorflow/core/api_def/base_api/api_def_Dequantize.pbtxt | 2 +- tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt | 2 +- tensorflow/go/op/wrappers.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Dequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_Dequantize.pbtxt index 40c00ef58f..cd4cc5c906 100644 --- a/tensorflow/core/api_def/base_api/api_def_Dequantize.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Dequantize.pbtxt @@ -21,7 +21,7 @@ used to convert the float values to their quantized equivalents. In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: ``` -if T == qint8, in[i] += (range(T) + 1)/ 2.0 +if T == qint8: in[i] += (range(T) + 1)/ 2.0 out[i] = min_range + (in[i]* (max_range - min_range) / range(T)) ``` here `range(T) = numeric_limits::max() - numeric_limits::min()` diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt index 37ac10dddb..b7311153f4 100644 --- a/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt @@ -42,7 +42,7 @@ In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: ``` out[i] = (in[i] - min_range) * range(T) / (max_range - min_range) -if T == qint8, out[i] -= (range(T) + 1) / 2.0 +if T == qint8: out[i] -= (range(T) + 1) / 2.0 ``` here `range(T) = numeric_limits::max() - numeric_limits::min()` diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 2f297d5161..9ec651777d 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -19500,7 +19500,7 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr { // // ``` // out[i] = (in[i] - min_range) * range(T) / (max_range - min_range) -// if T == qint8, out[i] -= (range(T) + 1) / 2.0 +// if T == qint8: out[i] -= (range(T) + 1) / 2.0 // ``` // // here `range(T) = numeric_limits::max() - numeric_limits::min()` @@ -23823,7 +23823,7 @@ func DequantizeMode(value string) DequantizeAttr { // In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: // // ``` -// if T == qint8, in[i] += (range(T) + 1)/ 2.0 +// if T == qint8: in[i] += (range(T) + 1)/ 2.0 // out[i] = min_range + (in[i]* (max_range - min_range) / range(T)) // ``` // here `range(T) = numeric_limits::max() - numeric_limits::min()` -- GitLab From 27489419e8d8870163f5173f77ea56aa118689d8 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Thu, 27 Sep 2018 23:46:02 -0700 Subject: [PATCH 0104/1085] Update the relative file pathes in the comments of tf.data kernel files --- tensorflow/core/kernels/data/batch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/cache_dataset_ops.cc | 2 +- tensorflow/core/kernels/data/concatenate_dataset_op.cc | 2 +- tensorflow/core/kernels/data/dataset_ops.cc | 2 +- .../core/kernels/data/dense_to_sparse_batch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/filter_by_component_dataset_op.cc | 2 +- tensorflow/core/kernels/data/filter_dataset_op.cc | 2 +- tensorflow/core/kernels/data/flat_map_dataset_op.cc | 2 +- tensorflow/core/kernels/data/generator_dataset_op.cc | 2 +- tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc | 2 +- tensorflow/core/kernels/data/group_by_window_dataset_op.cc | 2 +- tensorflow/core/kernels/data/interleave_dataset_op.cc | 2 +- tensorflow/core/kernels/data/iterator_ops.cc | 2 +- tensorflow/core/kernels/data/map_and_batch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/map_dataset_op.cc | 2 +- tensorflow/core/kernels/data/optimize_dataset_op.cc | 2 +- tensorflow/core/kernels/data/padded_batch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc | 2 +- tensorflow/core/kernels/data/parallel_map_dataset_op.cc | 2 +- tensorflow/core/kernels/data/parse_example_dataset_op.cc | 2 +- tensorflow/core/kernels/data/prefetch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/random_dataset_op.cc | 2 +- tensorflow/core/kernels/data/range_dataset_op.cc | 2 +- tensorflow/core/kernels/data/reader_dataset_ops.cc | 2 +- tensorflow/core/kernels/data/repeat_dataset_op.cc | 2 +- tensorflow/core/kernels/data/scan_dataset_op.cc | 2 +- tensorflow/core/kernels/data/shuffle_dataset_op.cc | 2 +- tensorflow/core/kernels/data/skip_dataset_op.cc | 2 +- tensorflow/core/kernels/data/slide_dataset_op.cc | 2 +- tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc | 2 +- tensorflow/core/kernels/data/sql_dataset_ops.cc | 2 +- tensorflow/core/kernels/data/take_dataset_op.cc | 2 +- tensorflow/core/kernels/data/tensor_dataset_op.cc | 2 +- tensorflow/core/kernels/data/tensor_slice_dataset_op.cc | 2 +- tensorflow/core/kernels/data/unbatch_dataset_op.cc | 2 +- tensorflow/core/kernels/data/window_dataset_op.cc | 2 +- tensorflow/core/kernels/data/zip_dataset_op.cc | 2 +- 37 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tensorflow/core/kernels/data/batch_dataset_op.cc b/tensorflow/core/kernels/data/batch_dataset_op.cc index d1db1d7bec..023cf79966 100644 --- a/tensorflow/core/kernels/data/batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/batch_dataset_op.cc @@ -21,7 +21,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class BatchDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc index 34c6c86538..d86d96b9fc 100644 --- a/tensorflow/core/kernels/data/cache_dataset_ops.cc +++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc @@ -23,7 +23,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level description of +// See documentation in ../../ops/dataset_ops.cc for a high-level description of // the following op. class CacheDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/concatenate_dataset_op.cc b/tensorflow/core/kernels/data/concatenate_dataset_op.cc index a04f150e71..46df039530 100644 --- a/tensorflow/core/kernels/data/concatenate_dataset_op.cc +++ b/tensorflow/core/kernels/data/concatenate_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ConcatenateDatasetOp : public BinaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/dataset_ops.cc b/tensorflow/core/kernels/data/dataset_ops.cc index bd1ccd5b5d..c689a119c3 100644 --- a/tensorflow/core/kernels/data/dataset_ops.cc +++ b/tensorflow/core/kernels/data/dataset_ops.cc @@ -21,7 +21,7 @@ limitations under the License. namespace tensorflow { namespace data { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class DatasetToGraphOp : public OpKernel { public: diff --git a/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc index 237511a07d..45678aa84f 100644 --- a/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc @@ -21,7 +21,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc index a7e3a56727..d09904a0eb 100644 --- a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc +++ b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. // TODO(prazek): Filter already has a logic of filtering by the given tensor, // but it must return both components. We could introduce kernel like diff --git a/tensorflow/core/kernels/data/filter_dataset_op.cc b/tensorflow/core/kernels/data/filter_dataset_op.cc index 00884314a9..a35f9a021c 100644 --- a/tensorflow/core/kernels/data/filter_dataset_op.cc +++ b/tensorflow/core/kernels/data/filter_dataset_op.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class FilterDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/flat_map_dataset_op.cc b/tensorflow/core/kernels/data/flat_map_dataset_op.cc index 2fada22a21..5d8565e745 100644 --- a/tensorflow/core/kernels/data/flat_map_dataset_op.cc +++ b/tensorflow/core/kernels/data/flat_map_dataset_op.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class FlatMapDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc index b4367d5a11..5de2e2871d 100644 --- a/tensorflow/core/kernels/data/generator_dataset_op.cc +++ b/tensorflow/core/kernels/data/generator_dataset_op.cc @@ -25,7 +25,7 @@ limitations under the License. namespace tensorflow { namespace data { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class GeneratorDatasetOp::Dataset : public DatasetBase { diff --git a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc b/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc index e7244ee208..87600d7873 100644 --- a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc +++ b/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc @@ -25,7 +25,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class GroupByReducerDatasetOp : public UnaryDatasetOpKernel { public: diff --git a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc index 14aefe5d54..7363664982 100644 --- a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc +++ b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { public: diff --git a/tensorflow/core/kernels/data/interleave_dataset_op.cc b/tensorflow/core/kernels/data/interleave_dataset_op.cc index 0aa802b874..83c1f7b719 100644 --- a/tensorflow/core/kernels/data/interleave_dataset_op.cc +++ b/tensorflow/core/kernels/data/interleave_dataset_op.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class InterleaveDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc index 7a833668ac..50b72f46c2 100644 --- a/tensorflow/core/kernels/data/iterator_ops.cc +++ b/tensorflow/core/kernels/data/iterator_ops.cc @@ -39,7 +39,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following ops. const char kIteratorVariantTypeName[] = "tensorflow::Iterator"; diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc index 2bbf4af664..b48a2c3eca 100644 --- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc @@ -34,7 +34,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc index f112e1dc43..1122d7918c 100644 --- a/tensorflow/core/kernels/data/map_dataset_op.cc +++ b/tensorflow/core/kernels/data/map_dataset_op.cc @@ -23,7 +23,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class MapDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index d5b725eac9..58d68d9de0 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -36,7 +36,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class OptimizeDatasetOp : public UnaryDatasetOpKernel { public: diff --git a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc index 7b01c3b4e0..d0943a583e 100644 --- a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc @@ -22,7 +22,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class PaddedBatchDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc index 2e6e0465f7..e180e510b7 100644 --- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc @@ -31,7 +31,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc index 6abe6c8338..c4e2fecc6e 100644 --- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc +++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc @@ -27,7 +27,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ParallelMapDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/parse_example_dataset_op.cc index c28c06da62..0d77dfe24e 100644 --- a/tensorflow/core/kernels/data/parse_example_dataset_op.cc +++ b/tensorflow/core/kernels/data/parse_example_dataset_op.cc @@ -23,7 +23,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ParseExampleDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc index 754ed772db..e1d42a9a6b 100644 --- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc +++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc @@ -26,7 +26,7 @@ limitations under the License. namespace tensorflow { namespace data { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class PrefetchDatasetOp::Dataset : public DatasetBase { diff --git a/tensorflow/core/kernels/data/random_dataset_op.cc b/tensorflow/core/kernels/data/random_dataset_op.cc index 044a791a3f..bcd26ab389 100644 --- a/tensorflow/core/kernels/data/random_dataset_op.cc +++ b/tensorflow/core/kernels/data/random_dataset_op.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class RandomDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/range_dataset_op.cc b/tensorflow/core/kernels/data/range_dataset_op.cc index 89fbaae369..0c0cb5ddc1 100644 --- a/tensorflow/core/kernels/data/range_dataset_op.cc +++ b/tensorflow/core/kernels/data/range_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class RangeDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/reader_dataset_ops.cc b/tensorflow/core/kernels/data/reader_dataset_ops.cc index c474cb4773..df4fbfc69a 100644 --- a/tensorflow/core/kernels/data/reader_dataset_ops.cc +++ b/tensorflow/core/kernels/data/reader_dataset_ops.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following ops. class TextLineDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/repeat_dataset_op.cc b/tensorflow/core/kernels/data/repeat_dataset_op.cc index 94e96635ab..e43dc1f6d8 100644 --- a/tensorflow/core/kernels/data/repeat_dataset_op.cc +++ b/tensorflow/core/kernels/data/repeat_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class RepeatDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/scan_dataset_op.cc b/tensorflow/core/kernels/data/scan_dataset_op.cc index 2a911aa368..c49a265b51 100644 --- a/tensorflow/core/kernels/data/scan_dataset_op.cc +++ b/tensorflow/core/kernels/data/scan_dataset_op.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ScanDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc index 66466d6a36..038d9cb9bd 100644 --- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc +++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc @@ -30,7 +30,7 @@ namespace { const int64 kLogIntervalMicros = 10 * 1000000; // 10 seconds. -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ShuffleDatasetOpBase : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/skip_dataset_op.cc b/tensorflow/core/kernels/data/skip_dataset_op.cc index b8c7fb15f4..bfaa632a74 100644 --- a/tensorflow/core/kernels/data/skip_dataset_op.cc +++ b/tensorflow/core/kernels/data/skip_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class SkipDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/slide_dataset_op.cc index 1e73cfc753..2be7fd7410 100644 --- a/tensorflow/core/kernels/data/slide_dataset_op.cc +++ b/tensorflow/core/kernels/data/slide_dataset_op.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class SlideDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc index 85b1e50695..ccb125f3c3 100644 --- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc @@ -24,7 +24,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. template diff --git a/tensorflow/core/kernels/data/sql_dataset_ops.cc b/tensorflow/core/kernels/data/sql_dataset_ops.cc index 6bbe459332..a50a041f5d 100644 --- a/tensorflow/core/kernels/data/sql_dataset_ops.cc +++ b/tensorflow/core/kernels/data/sql_dataset_ops.cc @@ -27,7 +27,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following ops. class SqlDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/take_dataset_op.cc b/tensorflow/core/kernels/data/take_dataset_op.cc index e5cdfdd732..e8570b68c9 100644 --- a/tensorflow/core/kernels/data/take_dataset_op.cc +++ b/tensorflow/core/kernels/data/take_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class TakeDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/tensor_dataset_op.cc b/tensorflow/core/kernels/data/tensor_dataset_op.cc index ca4ea25b89..ee0fb0069a 100644 --- a/tensorflow/core/kernels/data/tensor_dataset_op.cc +++ b/tensorflow/core/kernels/data/tensor_dataset_op.cc @@ -21,7 +21,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class TensorDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc index 7dc64b0a75..fe2f5ea536 100644 --- a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc @@ -22,7 +22,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class TensorSliceDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/unbatch_dataset_op.cc index 81c432b938..9d0abc21ef 100644 --- a/tensorflow/core/kernels/data/unbatch_dataset_op.cc +++ b/tensorflow/core/kernels/data/unbatch_dataset_op.cc @@ -21,7 +21,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class UnbatchDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/window_dataset_op.cc b/tensorflow/core/kernels/data/window_dataset_op.cc index ac44623ce2..16698c0b1a 100644 --- a/tensorflow/core/kernels/data/window_dataset_op.cc +++ b/tensorflow/core/kernels/data/window_dataset_op.cc @@ -22,7 +22,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class WindowDatasetOp : public UnaryDatasetOpKernel { diff --git a/tensorflow/core/kernels/data/zip_dataset_op.cc b/tensorflow/core/kernels/data/zip_dataset_op.cc index 61a2078f46..4186cb4ecd 100644 --- a/tensorflow/core/kernels/data/zip_dataset_op.cc +++ b/tensorflow/core/kernels/data/zip_dataset_op.cc @@ -20,7 +20,7 @@ namespace tensorflow { namespace data { namespace { -// See documentation in ../ops/dataset_ops.cc for a high-level +// See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following op. class ZipDatasetOp : public DatasetOpKernel { -- GitLab From d0690d46466bf0393ad65544d1e8c55e948df133 Mon Sep 17 00:00:00 2001 From: EFanZh Date: Fri, 28 Sep 2018 15:20:26 +0800 Subject: [PATCH 0105/1085] Fix some documentation errors --- tensorflow/contrib/distribute/python/mirrored_strategy.py | 5 +++-- tensorflow/python/keras/engine/training.py | 2 +- tensorflow/python/training/distribute.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py index 504f45a695..c0861da567 100644 --- a/tensorflow/contrib/distribute/python/mirrored_strategy.py +++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py @@ -318,12 +318,13 @@ class MirroredStrategy(distribute_lib.DistributionStrategy): [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed). The distribution strategy inherits these concepts as well and in addition to that we also clarify several more concepts: - * **In-graph replication**: the `client` creates a single `tf.Graph` that + + * **In-graph replication**: the `client` creates a single `tf.Graph` that specifies tasks for devices on all workers. The `client` then creates a client session which will talk to the `master` service of a `worker`. Then the `master` will partition the graph and distribute the work to all participating workers. - * **Worker**: A `worker` is a TensorFlow `task` that usually maps to one + * **Worker**: A `worker` is a TensorFlow `task` that usually maps to one physical machine. We will have multiple `worker`s with different `task` index. They all do similar things except for one worker checkpointing model variables, writing summaries, etc. in addition to its ordinary work. diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 5091cac836..1bd8422658 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -2356,6 +2356,6 @@ class DistributedCallbackModel(Model): # Whitelisted atttributes of the model that can be accessed by the user # during a callback. if item not in ['_setattr_tracking']: - logging.warning('You are accessing attribute ' + item + 'of the ' + logging.warning('You are accessing attribute ' + item + ' of the ' 'DistributedCallbackModel that may not have been set ' 'correctly.') diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py index 419a9ec12b..fd4704285c 100644 --- a/tensorflow/python/training/distribute.py +++ b/tensorflow/python/training/distribute.py @@ -631,7 +631,7 @@ class DistributionStrategy(object): Args: fn: function to run using this distribution strategy. The function must - have the following signature: def fn(context, *inputs). + have the following signature: `def fn(context, *inputs)`. `context` is an instance of `MultiStepContext` that will be passed when `fn` is run. `context` can be used to specify the outputs to be returned from `fn` by calling `context.set_last_step_output`. It can also be used @@ -797,9 +797,9 @@ class DistributionStrategy(object): return merged(results) ``` - Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`.' + Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`. - Neither *args nor **kwargs may contain per-device values. + Neither `*args` nor `**kwargs` may contain per-device values. If they contain mirrored values, they will be unwrapped before calling `fn`. -- GitLab From 8eb27871583d9fc61e046493acaa0df2839bc1c7 Mon Sep 17 00:00:00 2001 From: wangsiyu Date: Fri, 28 Sep 2018 18:51:34 +0800 Subject: [PATCH 0106/1085] remove slash --- tensorflow/python/ops/variables.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 69f63bc8e6..262cd61e5a 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -2401,7 +2401,8 @@ class PartitionedVariable(object): partition_axes = self._partition_axes() if len(partition_axes) > 1: raise NotImplementedError( - "Multi-axis partition assign_fn is not supported " + "Cannot do assign action along more than one dimension: %s. " + "Multi-axis partition assign action is not supported " % str(partition_axes)) partition_ix = partition_axes[0] size_splits_list = [ @@ -2409,7 +2410,7 @@ class PartitionedVariable(object): value_list = array_ops.split( value, size_splits_list, axis=partition_ix) op_list = [ - assign_fn(var, value_list[idx], idx) \ + assign_fn(var, value_list[idx], idx) for idx, var in enumerate(self._variable_list)] return op_list -- GitLab From a74a3217f7ff2dbee2fb618aa658cf666861545c Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Sat, 4 Aug 2018 14:13:00 +0800 Subject: [PATCH 0107/1085] Move bazel.rc to workspace root to support bazel-0.18.0 Bazel 0.18.0 will contain a change for which rc files it accepts. https://github.com/bazelbuild/bazel/commit/ec83598cb6ee4136166bb562a24dc5dfa58921db https://github.com/bazelbuild/bazel/issues/4502 Old bazel used to read %workspace%/tools/bazel.rc. New bazel will not read that and instead will only read %workspace%/.bazelrc. Signed-off-by: Jason Zaman --- tools/bazel.rc => .bazelrc | 4 +++- .gitignore | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) rename tools/bazel.rc => .bazelrc (98%) diff --git a/tools/bazel.rc b/.bazelrc similarity index 98% rename from tools/bazel.rc rename to .bazelrc index 3734fab715..9f09fdff97 100644 --- a/tools/bazel.rc +++ b/.bazelrc @@ -29,7 +29,7 @@ build:mkl -c opt # This config option is used to enable MKL-DNN open source library only, # without depending on MKL binary version. -build:mkl_open_source_only --define=build_with_mkl_dnn_only=true +build:mkl_open_source_only --define=build_with_mkl_dnn_only=true build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true build:download_clang --crosstool_top=@local_config_download_clang//:toolchain @@ -84,3 +84,5 @@ build:dynamic_kernels --define=dynamic_loaded_kernels=true build --define=PREFIX=/usr build --define=LIBDIR=$(PREFIX)/lib build --define=INCLUDEDIR=$(PREFIX)/include + +# Do not commit the tf_configure.bazelrc line diff --git a/.gitignore b/.gitignore index 1ef4c297ee..cb65f447d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ .DS_Store .ipynb_checkpoints node_modules -/.bazelrc /.tf_configure.bazelrc /bazel-* /bazel_pip -- GitLab From d3f6b72bc7356d5c94289e32426dc482b8ededf0 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Sat, 4 Aug 2018 14:28:02 +0800 Subject: [PATCH 0108/1085] configure: use workspace-relative path to tf_configure_bazelrc /.bazelrc is not gitignored anymore so this should help in case the import line is accidentally committed. Bazel 0.18.0 will support a new 'try-import' statement that should be used once 0.18.0 has been out long enough. Signed-off-by: Jason Zaman --- configure.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/configure.py b/configure.py index 55fce8b93b..129d9c5fe7 100644 --- a/configure.py +++ b/configure.py @@ -257,11 +257,7 @@ def reset_tf_configure_bazelrc(workspace_path): if _TF_BAZELRC_FILENAME in l: continue f.write('%s\n' % l) - if is_windows(): - tf_bazelrc_path = _TF_BAZELRC.replace('\\', '/') - else: - tf_bazelrc_path = _TF_BAZELRC - f.write('import %s\n' % tf_bazelrc_path) + f.write('import %%workspace%%/%s\n' % _TF_BAZELRC_FILENAME) def cleanup_makefile(): -- GitLab From b5feceb9058e06eac3de86ec45c44f5637054855 Mon Sep 17 00:00:00 2001 From: "Xiaoming (Jason) Cui" Date: Tue, 25 Sep 2018 00:42:42 -0700 Subject: [PATCH 0109/1085] Added the feature to disable MKL support of TensorFlow by environmental variable TF_DISABLE_MKL=1 --- .../core/common_runtime/mkl_cpu_allocator.h | 54 +++++++++++++------ .../core/common_runtime/process_util.cc | 5 ++ .../core/common_runtime/threadpool_device.cc | 4 ++ tensorflow/core/graph/mkl_layout_pass.cc | 5 ++ .../core/graph/mkl_tfconversion_pass.cc | 5 ++ tensorflow/core/util/util.cc | 20 +++++++ tensorflow/core/util/util.h | 5 ++ 7 files changed, 81 insertions(+), 17 deletions(-) diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h index 429b19599b..516138d28d 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/mem.h" +#include "tensorflow/core/util/util.h" #include "tensorflow/core/platform/numa.h" #ifndef INTEL_MKL_DNN_ONLY @@ -163,6 +164,12 @@ class MklCPUAllocator : public Allocator { } Status Initialize() { + if (DisableMKL()) { + VLOG(1) << "TF-MKL: Disabling pool allocator"; + tf_disable_pool_allocator_flag_ = true; + return Status::OK(); + } + VLOG(2) << "MklCPUAllocator: In MklCPUAllocator"; // Set upper bound on memory allocation to physical RAM available on the @@ -217,6 +224,10 @@ class MklCPUAllocator : public Allocator { inline string Name() override { return kName; } inline void* AllocateRaw(size_t alignment, size_t num_bytes) override { + if (tf_disable_pool_allocator_flag_) { + return port::AlignedMalloc(num_bytes, alignment); + } + // If the allocation size is less than threshold, call small allocator, // otherwise call large-size allocator (BFC). We found that BFC allocator // does not deliver good performance for small allocations when @@ -227,6 +238,10 @@ class MklCPUAllocator : public Allocator { } inline void DeallocateRaw(void* ptr) override { + if (tf_disable_pool_allocator_flag_) { + port::AlignedFree(ptr); + return; + } // Check if ptr is for "small" allocation. If it is, then call Free // directly. Otherwise, call BFC to handle free. if (small_size_allocator_->IsSmallSizeAllocation(ptr)) { @@ -237,26 +252,30 @@ class MklCPUAllocator : public Allocator { } void GetStats(AllocatorStats* stats) override { - AllocatorStats l_stats, s_stats; - small_size_allocator_->GetStats(&s_stats); - large_size_allocator_->GetStats(&l_stats); - - // Combine statistics from small-size and large-size allocator. - stats->num_allocs = l_stats.num_allocs + s_stats.num_allocs; - stats->bytes_in_use = l_stats.bytes_in_use + s_stats.bytes_in_use; - stats->max_bytes_in_use = - l_stats.max_bytes_in_use + s_stats.max_bytes_in_use; - - // Since small-size allocations go to MklSmallSizeAllocator, - // max_alloc_size from large_size_allocator would be the maximum - // size allocated by MklCPUAllocator. - stats->max_alloc_size = l_stats.max_alloc_size; - stats->bytes_limit = std::max(s_stats.bytes_limit, l_stats.bytes_limit); + if (!tf_disable_pool_allocator_flag_) { + AllocatorStats l_stats, s_stats; + small_size_allocator_->GetStats(&s_stats); + large_size_allocator_->GetStats(&l_stats); + + // Combine statistics from small-size and large-size allocator. + stats->num_allocs = l_stats.num_allocs + s_stats.num_allocs; + stats->bytes_in_use = l_stats.bytes_in_use + s_stats.bytes_in_use; + stats->max_bytes_in_use = + l_stats.max_bytes_in_use + s_stats.max_bytes_in_use; + + // Since small-size allocations go to MklSmallSizeAllocator, + // max_alloc_size from large_size_allocator would be the maximum + // size allocated by MklCPUAllocator. + stats->max_alloc_size = l_stats.max_alloc_size; + stats->bytes_limit = std::max(s_stats.bytes_limit, l_stats.bytes_limit); + } } void ClearStats() override { - small_size_allocator_->ClearStats(); - large_size_allocator_->ClearStats(); + if (!tf_disable_pool_allocator_flag_) { + small_size_allocator_->ClearStats(); + large_size_allocator_->ClearStats(); + } } private: @@ -295,6 +314,7 @@ class MklCPUAllocator : public Allocator { // The alignment that we need for the allocations static constexpr const size_t kAlignment = 64; + bool tf_disable_pool_allocator_flag_ = false; Allocator* large_size_allocator_; // owned by this class MklSmallSizeAllocator* small_size_allocator_; // owned by this class. diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index a5d31b75c7..60fa601907 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/tracing.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/util.h" namespace tensorflow { @@ -56,6 +57,10 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) { const int32 inter_op = options.config.inter_op_parallelism_threads(); if (inter_op != 0) return inter_op; #ifdef INTEL_MKL + // Early return if MKL is disabled + if (DisableMKL()) + return port::NumSchedulableCPUs(); + // MKL library executes ops in parallel using OMP threads // Set inter_op conservatively to avoid thread oversubscription that could // lead to severe perf degradations and OMP resource exhaustion diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index 8587d1783a..29c01d7f72 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/platform/tracing.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/util/util.h" #ifdef INTEL_MKL #ifdef _OPENMP @@ -49,6 +50,9 @@ ThreadPoolDevice::ThreadPoolDevice(const SessionOptions& options, allocator_(allocator), scoped_allocator_mgr_(new ScopedAllocatorMgr(name)) { #ifdef INTEL_MKL + // Eearly return when MKL is disabled + if (DisableMKL()) + return; #ifdef _OPENMP const char* user_omp_threads = getenv("OMP_NUM_THREADS"); if (user_omp_threads == nullptr) { diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc index 06d3fefef1..7394b1cddf 100644 --- a/tensorflow/core/graph/mkl_layout_pass.cc +++ b/tensorflow/core/graph/mkl_layout_pass.cc @@ -38,6 +38,7 @@ limitations under the License. #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/util/tensor_format.h" +#include "tensorflow/core/util/util.h" #include "tensorflow/core/graph/mkl_graph_util.h" #include "tensorflow/core/graph/mkl_layout_pass.h" @@ -4511,6 +4512,10 @@ Status MklLayoutRewritePass::Run(const GraphOptimizationPassOptions& options) { if (options.graph == nullptr && options.partition_graphs == nullptr) { return Status::OK(); } + if (DisableMKL()) { + VLOG(2) << "TF-MKL: Disabling MKL"; + return Status::OK(); + } auto process_graph = [&](std::unique_ptr* g) { // Get the ownership of a graph diff --git a/tensorflow/core/graph/mkl_tfconversion_pass.cc b/tensorflow/core/graph/mkl_tfconversion_pass.cc index 8c5ffd71a3..6804ab84ce 100644 --- a/tensorflow/core/graph/mkl_tfconversion_pass.cc +++ b/tensorflow/core/graph/mkl_tfconversion_pass.cc @@ -31,6 +31,7 @@ limitations under the License. #include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/util.h" #include "tensorflow/core/graph/mkl_graph_util.h" #include "tensorflow/core/graph/mkl_tfconversion_pass.h" @@ -424,6 +425,10 @@ Status MklToTfConversionPass::Run(const GraphOptimizationPassOptions& options) { if (options.graph == nullptr && options.partition_graphs == nullptr) { return Status::OK(); } + if (DisableMKL()) { + VLOG(2) << "TF-MKL: Disabling MKL"; + return Status::OK(); + } auto process_graph = [&](std::unique_ptr* g) { // Get the ownership of graph diff --git a/tensorflow/core/util/util.cc b/tensorflow/core/util/util.cc index 1e5a9c5712..44d5becb9c 100644 --- a/tensorflow/core/util/util.cc +++ b/tensorflow/core/util/util.cc @@ -120,4 +120,24 @@ string SliceDebugString(const TensorShape& shape, const int64 flat) { return result; } +#ifdef INTEL_MKL +bool DisableMKL() { + enum MklStatus { + MKL_DEFAULT = 0, + MKL_ON = 1, + MKL_OFF = 2 + }; + static MklStatus status = MKL_DEFAULT; + if (status == MKL_DEFAULT) { + char* tf_disable_mkl = getenv("TF_DISABLE_MKL"); + if ((tf_disable_mkl != NULL) && (std::stoi(tf_disable_mkl) == 1)) { + VLOG(2) << "TF-MKL: Disabling MKL"; + status = MKL_OFF; + } else { + status = MKL_ON; + } + } + return status == MKL_OFF ? true : false; +} +#endif } // namespace tensorflow diff --git a/tensorflow/core/util/util.h b/tensorflow/core/util/util.h index 93dfd51ab5..ba90ad52c2 100644 --- a/tensorflow/core/util/util.h +++ b/tensorflow/core/util/util.h @@ -56,6 +56,11 @@ string PrintMemory(const char* ptr, size_t n); // "tensor", "tensor[i]", "tensor[i, j]", etc. string SliceDebugString(const TensorShape& shape, const int64 flat); +// disable MKL in runtime +#ifdef INTEL_MKL +bool DisableMKL(); +#endif + } // namespace tensorflow #endif // TENSORFLOW_CORE_UTIL_UTIL_H_ -- GitLab From 921f8347bd463347653389f65803326528609691 Mon Sep 17 00:00:00 2001 From: franklin5 Date: Fri, 28 Sep 2018 15:12:46 -0700 Subject: [PATCH 0110/1085] attention score shape from equation 4, the attention score should not be a rank-3 tensor object. --- .../python/examples/nmt_with_attention/nmt_with_attention.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb index 560fc8c5a2..6d463cc5c4 100644 --- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb +++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb @@ -352,7 +352,7 @@ "And the pseudo-code:\n", "\n", "* `score = FC(tanh(FC(EO) + FC(H)))`\n", - "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, hidden_size)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n", + "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n", "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n", "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n", "* `merged vector = concat(embedding output, context vector)`\n", -- GitLab From ba2d36ef87cfb6cbbf06abd998edbd1ad047a741 Mon Sep 17 00:00:00 2001 From: franklin5 Date: Fri, 28 Sep 2018 16:10:26 -0700 Subject: [PATCH 0111/1085] updating code to reflect on pseudo code and equations --- .../examples/nmt_with_attention/nmt_with_attention.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb index 6d463cc5c4..480777d948 100644 --- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb +++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb @@ -446,12 +446,12 @@ " # we are doing this to perform addition to calculate the score\n", " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", " \n", - " # score shape == (batch_size, max_length, hidden_size)\n", - " score = tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis))\n", + " # score shape == (batch_size, max_length, 1)\n", + " # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n", + " score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n", " \n", " # attention_weights shape == (batch_size, max_length, 1)\n", - " # we get 1 at the last axis because we are applying score to self.V\n", - " attention_weights = tf.nn.softmax(self.V(score), axis=1)\n", + " attention_weights = tf.nn.softmax(score, axis=1)\n", " \n", " # context_vector shape after sum == (batch_size, hidden_size)\n", " context_vector = attention_weights * enc_output\n", -- GitLab From e4fea9419ac387ddcb9c932abaa8e92fb045e29f Mon Sep 17 00:00:00 2001 From: knightXun Date: Sat, 29 Sep 2018 00:42:23 +0800 Subject: [PATCH 0112/1085] print error information, when the os is not supported --- tensorflow/go/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/go/test.sh b/tensorflow/go/test.sh index 6083608f22..47c3a68379 100755 --- a/tensorflow/go/test.sh +++ b/tensorflow/go/test.sh @@ -63,6 +63,9 @@ then else export DYLD_LIBRARY_PATH="${PWD}/tensorflow:${DYLD_LIBRARY_PATH}" fi +else + echo "Only support Linux/Darwin, System $OS is not supported" + exit 1 fi # Document the Go version and run tests -- GitLab From d936d819752916d3122f02def571ecac9e995029 Mon Sep 17 00:00:00 2001 From: "Xiaoming (Jason) Cui" Date: Fri, 28 Sep 2018 19:49:23 -0700 Subject: [PATCH 0113/1085] Lower the MKLCpuAllocator priority so that it can use default allocator when MKL is disabled, and with some minor changes --- .../core/common_runtime/mkl_cpu_allocator.h | 54 ++++++------------- .../core/common_runtime/process_util.cc | 37 ++++++------- .../core/common_runtime/threadpool_device.cc | 4 +- 3 files changed, 36 insertions(+), 59 deletions(-) diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h index 516138d28d..429b19599b 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h @@ -27,7 +27,6 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/mem.h" -#include "tensorflow/core/util/util.h" #include "tensorflow/core/platform/numa.h" #ifndef INTEL_MKL_DNN_ONLY @@ -164,12 +163,6 @@ class MklCPUAllocator : public Allocator { } Status Initialize() { - if (DisableMKL()) { - VLOG(1) << "TF-MKL: Disabling pool allocator"; - tf_disable_pool_allocator_flag_ = true; - return Status::OK(); - } - VLOG(2) << "MklCPUAllocator: In MklCPUAllocator"; // Set upper bound on memory allocation to physical RAM available on the @@ -224,10 +217,6 @@ class MklCPUAllocator : public Allocator { inline string Name() override { return kName; } inline void* AllocateRaw(size_t alignment, size_t num_bytes) override { - if (tf_disable_pool_allocator_flag_) { - return port::AlignedMalloc(num_bytes, alignment); - } - // If the allocation size is less than threshold, call small allocator, // otherwise call large-size allocator (BFC). We found that BFC allocator // does not deliver good performance for small allocations when @@ -238,10 +227,6 @@ class MklCPUAllocator : public Allocator { } inline void DeallocateRaw(void* ptr) override { - if (tf_disable_pool_allocator_flag_) { - port::AlignedFree(ptr); - return; - } // Check if ptr is for "small" allocation. If it is, then call Free // directly. Otherwise, call BFC to handle free. if (small_size_allocator_->IsSmallSizeAllocation(ptr)) { @@ -252,30 +237,26 @@ class MklCPUAllocator : public Allocator { } void GetStats(AllocatorStats* stats) override { - if (!tf_disable_pool_allocator_flag_) { - AllocatorStats l_stats, s_stats; - small_size_allocator_->GetStats(&s_stats); - large_size_allocator_->GetStats(&l_stats); - - // Combine statistics from small-size and large-size allocator. - stats->num_allocs = l_stats.num_allocs + s_stats.num_allocs; - stats->bytes_in_use = l_stats.bytes_in_use + s_stats.bytes_in_use; - stats->max_bytes_in_use = - l_stats.max_bytes_in_use + s_stats.max_bytes_in_use; - - // Since small-size allocations go to MklSmallSizeAllocator, - // max_alloc_size from large_size_allocator would be the maximum - // size allocated by MklCPUAllocator. - stats->max_alloc_size = l_stats.max_alloc_size; - stats->bytes_limit = std::max(s_stats.bytes_limit, l_stats.bytes_limit); - } + AllocatorStats l_stats, s_stats; + small_size_allocator_->GetStats(&s_stats); + large_size_allocator_->GetStats(&l_stats); + + // Combine statistics from small-size and large-size allocator. + stats->num_allocs = l_stats.num_allocs + s_stats.num_allocs; + stats->bytes_in_use = l_stats.bytes_in_use + s_stats.bytes_in_use; + stats->max_bytes_in_use = + l_stats.max_bytes_in_use + s_stats.max_bytes_in_use; + + // Since small-size allocations go to MklSmallSizeAllocator, + // max_alloc_size from large_size_allocator would be the maximum + // size allocated by MklCPUAllocator. + stats->max_alloc_size = l_stats.max_alloc_size; + stats->bytes_limit = std::max(s_stats.bytes_limit, l_stats.bytes_limit); } void ClearStats() override { - if (!tf_disable_pool_allocator_flag_) { - small_size_allocator_->ClearStats(); - large_size_allocator_->ClearStats(); - } + small_size_allocator_->ClearStats(); + large_size_allocator_->ClearStats(); } private: @@ -314,7 +295,6 @@ class MklCPUAllocator : public Allocator { // The alignment that we need for the allocations static constexpr const size_t kAlignment = 64; - bool tf_disable_pool_allocator_flag_ = false; Allocator* large_size_allocator_; // owned by this class MklSmallSizeAllocator* small_size_allocator_; // owned by this class. diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index 60fa601907..b3064a4c08 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -57,28 +57,25 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) { const int32 inter_op = options.config.inter_op_parallelism_threads(); if (inter_op != 0) return inter_op; #ifdef INTEL_MKL - // Early return if MKL is disabled - if (DisableMKL()) - return port::NumSchedulableCPUs(); - - // MKL library executes ops in parallel using OMP threads - // Set inter_op conservatively to avoid thread oversubscription that could - // lead to severe perf degradations and OMP resource exhaustion - int mkl_intra_op = 1; -#ifdef _OPENMP - mkl_intra_op = omp_get_max_threads(); -#endif // _OPENMP - CHECK_GE(mkl_intra_op, 1); - const int32 mkl_inter_op = std::max( - (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2); - VLOG(0) << "Creating new thread pool with default inter op setting: " - << mkl_inter_op - << ". Tune using inter_op_parallelism_threads for best performance."; - return mkl_inter_op; -#else + if (!DisableMKL()) { + // MKL library executes ops in parallel using OMP threads + // Set inter_op conservatively to avoid thread oversubscription that could + // lead to severe perf degradations and OMP resource exhaustion + int mkl_intra_op = 1; + #ifdef _OPENMP + mkl_intra_op = omp_get_max_threads(); + #endif // _OPENMP + CHECK_GE(mkl_intra_op, 1); + const int32 mkl_inter_op = std::max( + (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2); + VLOG(0) << "Creating new thread pool with default inter op setting: " + << mkl_inter_op + << ". Tune using inter_op_parallelism_threads for best performance."; + return mkl_inter_op; + } +#endif // INTEL_MKL // Default to using the number of cores available in the process. return port::NumSchedulableCPUs(); -#endif // INTEL_MKL } thread::ThreadPool* NewThreadPoolFromSessionOptions( diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index 29c01d7f72..f188016610 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -50,7 +50,7 @@ ThreadPoolDevice::ThreadPoolDevice(const SessionOptions& options, allocator_(allocator), scoped_allocator_mgr_(new ScopedAllocatorMgr(name)) { #ifdef INTEL_MKL - // Eearly return when MKL is disabled + // Early return when MKL is disabled if (DisableMKL()) return; #ifdef _OPENMP @@ -118,7 +118,7 @@ class MklCPUAllocatorFactory : public AllocatorFactory { }; #ifdef ENABLE_MKL -REGISTER_MEM_ALLOCATOR("MklCPUAllocator", 200, MklCPUAllocatorFactory); +REGISTER_MEM_ALLOCATOR("MklCPUAllocator", (DisableMKL() ? 50 : 200), MklCPUAllocatorFactory); #endif // ENABLE_MKL } // namespace -- GitLab From 04865e685f9d8ca8723e8f4e4aee3da41f902ae6 Mon Sep 17 00:00:00 2001 From: lanhin Date: Sat, 29 Sep 2018 20:48:07 +0800 Subject: [PATCH 0114/1085] Comment fix: cudnn output tensor data_format convert. --- tensorflow/core/kernels/conv_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index 78856c4a99..4e60d9aada 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -851,7 +851,7 @@ void LaunchConv2DOp::operator()( ") filter shape(", filter.shape().DebugString(), ")")); } - // Convert the output tensor back from NHWC to NCHW. + // Convert the output tensor back from NCHW to NHWC. if (data_format == FORMAT_NHWC) { functor::NCHWToNHWC()( ctx->eigen_device(), -- GitLab From f16111286b19f4145df63b73c45be1645bde8737 Mon Sep 17 00:00:00 2001 From: Bairen Yi Date: Sat, 29 Sep 2018 22:13:09 +0800 Subject: [PATCH 0115/1085] Added log entries for copying unpinned memory RDMA Currently there are large number of tensors managed by non-visitable memory allocators in CPU-only PS. GPU workers seem less prone to this problem. Copying large sized tensor buffers may introduce non-trivial overhead. Should probably fix this. Signed-off-by: Bairen Yi --- tensorflow/contrib/gdr/gdr_memory_manager.cc | 156 +++++++++++-------- 1 file changed, 93 insertions(+), 63 deletions(-) diff --git a/tensorflow/contrib/gdr/gdr_memory_manager.cc b/tensorflow/contrib/gdr/gdr_memory_manager.cc index bb06f1c41c..3549cedb70 100644 --- a/tensorflow/contrib/gdr/gdr_memory_manager.cc +++ b/tensorflow/contrib/gdr/gdr_memory_manager.cc @@ -22,7 +22,6 @@ limitations under the License. #include #include #include -#include #include #include @@ -30,19 +29,17 @@ limitations under the License. #include #include "tensorflow/contrib/gdr/gdr.pb.h" -#include "tensorflow/core/common_runtime/bfc_allocator.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/dma_helper.h" -#include "tensorflow/core/common_runtime/pool_allocator.h" #include "tensorflow/core/common_runtime/process_state.h" #if GOOGLE_CUDA #include "tensorflow/core/common_runtime/gpu/gpu_process_state.h" #include "tensorflow/core/common_runtime/gpu/gpu_util.h" #endif // GOOGLE_CUDA -#include "tensorflow/core/framework/allocator_registry.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/numa.h" namespace tensorflow { @@ -70,14 +67,11 @@ bool IsGDRAvailable() { int TryToReadNumaNode(ibv_device* device) { #if defined(__APPLE__) LOG(INFO) << "OS X does not support NUMA - returning NUMA node 0"; - return 0; + return port::kNUMANoAffinity; #elif defined(PLATFORM_WINDOWS) // Windows support for NUMA is not currently implemented. Return node 0. - return 0; + return port::kNUMANoAffinity; #else - VLOG(2) << "Trying to read NUMA node for device: " << device->name; - static const int kUnknownNumaNode = -1; - auto filename = string(device->ibdev_path) + "/device/numa_node"; std::ifstream ifs(filename.c_str()); @@ -91,12 +85,12 @@ int TryToReadNumaNode(ibv_device* device) { << value << "), but there must be at least one NUMA node" ", so returning NUMA node zero"; - return 0; + return port::kNUMANoAffinity; } LOG(INFO) << "NUMA node for device: " << device->name << " is " << value; return value; } - return kUnknownNumaNode; + return port::kNUMANoAffinity; #endif } @@ -138,8 +132,6 @@ class GdrMemoryManager : public RemoteMemoryManager { Device* device, DeviceContext* device_context, bool on_host, StatusCallback done) override; - static void RegMemVisitors(); - protected: Status CreateEndpoint(const string& host, const string& port, RdmaEndpointPtr& endpoint); @@ -150,7 +142,8 @@ class GdrMemoryManager : public RemoteMemoryManager { ibv_mr* FindMemoryRegion(void* addr, size_t length); - void InsertMemoryRegion(void* addr, size_t length); + void InsertMemoryRegion(void* addr, size_t length, + const std::string& allocator_name); void EvictMemoryRegion(void* addr, size_t length); @@ -160,6 +153,7 @@ class GdrMemoryManager : public RemoteMemoryManager { RdmaEndpointPtr listening_; std::atomic stopped_; int epfd_; + int numa_node_; // Server side endpoints // Accessed sequentially in Run() so not protected by lock @@ -190,46 +184,10 @@ GdrMemoryManager::GdrMemoryManager(const string& host, const string& port) port_(port), listening_(nullptr, EndpointDeleter), stopped_(true), - next_key_(0) { - static std::once_flag flag; - std::call_once(flag, []() { RegMemVisitors(); }); -} + next_key_(0) {} GdrMemoryManager::~GdrMemoryManager() { close(epfd_); } -/*static*/ void GdrMemoryManager::RegMemVisitors() { - SubAllocator::Visitor alloc_visitor = [](void* ptr, int numa_node, - size_t num_bytes) { - GdrMemoryManager::Singleton().InsertMemoryRegion( - ptr, num_bytes, strings::StrCat("CPU:", numa_node)); - }; - SubAllocator::Visitor free_visitor = [](void* ptr, int numa_node, - size_t num_bytes) { - GdrMemoryManager::Singleton().EvictMemoryRegion(ptr, num_bytes); - }; - ProcessState::singleton()->AddCPUAllocVisitor(alloc_visitor); - ProcessState::singleton()->AddCPUFreeVisitor(free_visitor); - -#if GOOGLE_CUDA - if (IsGDRAvailable()) { - int32_t bus_id = TryToReadNumaNode(rdma_adapter_->context_->device) + 1; - - // Note we don't free allocated GPU memory so there is no free visitor - SubAllocator::Visitor cuda_alloc_visitor = [](void* ptr, int gpu_id, - size_t num_bytes) { - RdmaMemoryMgr::Singleton().InsertMemoryRegion( - ptr, num_bytes, strings::StrCat("GPU:", gpu_id)); - }; - GPUProcessState::singleton()->AddGPUAllocVisitor(bus_id, - cuda_alloc_visitor); - GPUProcessState::singleton()->AddCUDAHostAllocVisitor(bus_id, - alloc_visitor); - GPUProcessState::singleton()->AddCUDAHostFreeVisitor(bus_id, free_visitor); - LOG(INFO) << "Instrumenting GPU allocator with bus_id " << bus_id; - } -#endif // GOOGLE_CUDA -} - Status GdrMemoryManager::Init() { epfd_ = epoll_create1(0); if (epfd_ == -1) { @@ -289,6 +247,42 @@ Status GdrMemoryManager::Init() { "cannot add server to epoll"); } + numa_node_ = TryToReadNumaNode(listening_->verbs->device); + + SubAllocator::Visitor alloc_visitor = [this](void* ptr, int numa_node, + size_t num_bytes) { + VLOG(2) << "Registering RDMA capable memory region on numa_node " + << numa_node; + InsertMemoryRegion(ptr, num_bytes, strings::StrCat("CPU:", numa_node)); + }; + SubAllocator::Visitor free_visitor = [this](void* ptr, int numa_node, + size_t num_bytes) { + VLOG(2) << "De-registering RDMA capable memory region on numa_node " + << numa_node; + EvictMemoryRegion(ptr, num_bytes); + }; + ProcessState::singleton()->AddCPUAllocVisitor(alloc_visitor); + ProcessState::singleton()->AddCPUFreeVisitor(free_visitor); + LOG(INFO) << "Instrumenting CPU allocator(s)"; + +#if GOOGLE_CUDA + if (IsGDRAvailable()) { + int bus_id = numa_node_ + 1; + + SubAllocator::Visitor cuda_alloc_visitor = [this](void* ptr, int gpu_id, + size_t num_bytes) { + VLOG(2) << "Registering RDMA capable memory region on GPU " << gpu_id; + InsertMemoryRegion(ptr, num_bytes, strings::StrCat("GPU:", gpu_id)); + }; + GPUProcessState::singleton()->AddGPUAllocVisitor(bus_id, + cuda_alloc_visitor); + GPUProcessState::singleton()->AddCUDAHostAllocVisitor(bus_id, + alloc_visitor); + GPUProcessState::singleton()->AddCUDAHostFreeVisitor(bus_id, free_visitor); + LOG(INFO) << "Instrumenting GPU allocator(s) with bus_id " << bus_id; + } +#endif // GOOGLE_CUDA + return Status::OK(); } @@ -405,7 +399,7 @@ void GdrMemoryManager::TransportOptionsFromTensor( ibv_mr* mr = FindMemoryRegion(addr, length); #if GOOGLE_CUDA - if (!on_host) { + if (device->tensorflow_gpu_device_info() && !on_host) { Allocator* alloc = GPUProcessState::singleton()->GetCUDAHostAllocator(0); Tensor* host_copy = new Tensor(alloc, tensor.dtype(), tensor.shape()); GPUUtil::CopyGPUTensorToCPU( @@ -456,11 +450,27 @@ void GdrMemoryManager::TransportOptionsFromTensor( #endif if (mr == nullptr) { - done(errors::Unavailable("Cannot find pinned memory region")); - return; + Allocator* alloc = ProcessState::singleton()->GetCPUAllocator(numa_node_); + Tensor host_copy(alloc, tensor.dtype(), tensor.shape()); + + std::memcpy(DMAHelper::buffer(&host_copy)->data(), buffer->data(), length); + VLOG(2) << "Copying " << length << " bytes unpinned tensor buffer"; + + buffer = DMAHelper::buffer(&host_copy); + addr = buffer->data(); + length = buffer->size(); + + mr = FindMemoryRegion(addr, length); + if (mr == nullptr) { + done(errors::Unavailable("Cannot find pinned memory region")); + return; + } + + buffer->Ref(); + } else { + buffer->Ref(); } - buffer->Ref(); TensorKey tensor_key = next_key_++; { mutex_lock l(server_mu_); @@ -470,7 +480,7 @@ void GdrMemoryManager::TransportOptionsFromTensor( uint64_t checksum = 0; if (VLOG_IS_ON(2)) { #ifdef GOOGLE_CUDA - if (!on_host) { + if (device->tensorflow_gpu_device_info() && !on_host) { checksum = GPUUtil::Checksum(device, device_context, tensor); } else { checksum = GPUUtil::Checksum(tensor); @@ -508,7 +518,8 @@ void GdrMemoryManager::TensorFromTransportOptions( Tensor host_copy; #if GOOGLE_CUDA if (mr == nullptr && !on_host) { - Allocator* alloc = GPUProcessState::singleton()->GetCUDAHostAllocator(0); + Allocator* alloc = + GPUProcessState::singleton()->GetCUDAHostAllocator(numa_node_); host_copy = Tensor(alloc, tensor->dtype(), tensor->shape()); buffer = DMAHelper::buffer(&host_copy); addr = buffer->data(); @@ -518,8 +529,18 @@ void GdrMemoryManager::TensorFromTransportOptions( #endif // GOOGLE_CUDA if (mr == nullptr) { - done(errors::Unavailable("Cannot find pinned memory region")); - return; + Allocator* alloc = ProcessState::singleton()->GetCPUAllocator(numa_node_); + host_copy = Tensor(alloc, tensor->dtype(), tensor->shape()); + + buffer = DMAHelper::buffer(&host_copy); + addr = buffer->data(); + length = buffer->size(); + + mr = FindMemoryRegion(addr, length); + if (mr == nullptr) { + done(errors::Unavailable("Cannot find pinned memory region")); + return; + } } decltype(clients_)::iterator iter; @@ -568,7 +589,8 @@ void GdrMemoryManager::TensorFromTransportOptions( } #if GOOGLE_CUDA - if (host_copy.NumElements() > 0) { + if (device->tensorflow_gpu_device_info() && !on_host && + host_copy.NumElements() > 0) { uint64_t checksum = 0; if (VLOG_IS_ON(2)) { checksum = GPUUtil::Checksum(host_copy); @@ -598,6 +620,12 @@ void GdrMemoryManager::TensorFromTransportOptions( } #endif // GOOGLE_CUDA + if ((on_host || !device->tensorflow_gpu_device_info()) && + host_copy.NumElements() > 0) { + std::memcpy(DMAHelper::buffer(tensor)->data(), addr, length); + VLOG(2) << "Copying " << length << " bytes unpinned tensor buffer"; + } + uint64_t end = Env::Default()->NowMicros(); VLOG(2) << "RDMA from remote memory region " << remote_mr.rkey() @@ -607,7 +635,7 @@ void GdrMemoryManager::TensorFromTransportOptions( uint64_t checksum = 0; if (VLOG_IS_ON(2)) { #ifdef GOOGLE_CUDA - if (device->tensorflow_gpu_device_info() && (!on_host)) { + if (device->tensorflow_gpu_device_info() && !on_host) { checksum = GPUUtil::Checksum(device, device_context, *tensor); } else { checksum = GPUUtil::Checksum(*tensor); @@ -668,7 +696,8 @@ ibv_mr* GdrMemoryManager::FindMemoryRegion(void* addr, size_t length) { } } -void GdrMemoryManager::InsertMemoryRegion(void* addr, size_t length) { +void GdrMemoryManager::InsertMemoryRegion(void* addr, size_t length, + const std::string& allocator_name) { if (length == 0) return; ibv_mr* mr = rdma_reg_read(listening_.get(), addr, length); if (mr != nullptr) { @@ -676,7 +705,8 @@ void GdrMemoryManager::InsertMemoryRegion(void* addr, size_t length) { auto iter = std::upper_bound(mrs_.begin(), mrs_.end(), addr, &Comparator); mrs_.insert(iter, {mr, &MRDeleter}); } else { - LOG(WARNING) << "Cannot register memory region"; + LOG(WARNING) << "Cannot register memory region allocated by " + << allocator_name; } } -- GitLab From eb6c1bdcbf6093888f2b443fdb49f836f3352316 Mon Sep 17 00:00:00 2001 From: Joe Yearsley Date: Tue, 13 Mar 2018 07:23:18 +0000 Subject: [PATCH 0116/1085] Update core.py Added `data_format` to flatten to allow changing of it during inference time. --- tensorflow/python/layers/core.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index 9879e5020f..5f89e3c0c3 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -268,7 +268,14 @@ def dropout(inputs, @tf_export('layers.Flatten') class Flatten(keras_layers.Flatten, base.Layer): """Flattens an input tensor while preserving the batch axis (axis 0). - + + Arguments: + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + Examples: ``` @@ -285,11 +292,16 @@ class Flatten(keras_layers.Flatten, base.Layer): @tf_export('layers.flatten') -def flatten(inputs, name=None): +def flatten(inputs, data_format='channels_last', name=None): """Flattens an input tensor while preserving the batch axis (axis 0). Arguments: inputs: Tensor input. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, height, width, channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, height, width)`. name: The name of the layer (string). Returns: @@ -307,7 +319,7 @@ def flatten(inputs, name=None): # now `y` has shape `(None, None)` ``` """ - layer = Flatten(name=name) + layer = Flatten(data_format=data_format, name=name) return layer.apply(inputs) -- GitLab From dd928d5ae31dd0484e5e4a96c6322adecc4e511b Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Sun, 18 Mar 2018 19:24:10 +0000 Subject: [PATCH 0117/1085] Added Flatten Test --- tensorflow/python/layers/core_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index d26f3f4789..0d019897aa 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -476,6 +476,22 @@ class FlattenTest(test.TestCase): shape = core_layers.Flatten().compute_output_shape((None, 3, None)) self.assertEqual(shape.as_list(), [None, None]) + def testDataFormat(self): + np_input_channels_last = np.arange(3, 7).reshape([1, 2, 3, 2]) + + with self.test_session() as sess: + x = array_ops.placeholder(shape=(1, 2, 3, 2), dtype='float32') + y = core_layers.Flatten(data_format='channels_last')(x) + np_output_cl = sess.run(y, feed_dict={x: np_input_channels_last}) + + x = array_ops.placeholder(shape=(1, 2, 3, 2), dtype='float32') + y = core_layers.Flatten(data_format='channels_first')(x) + np_input_channels_first = np.transpose(np_input_channels_last, + [0, 3, 1, 2]) + np_output_cf = sess.run(y, feed_dict={x: np_input_channels_first}) + + self.assertEqual(np_output_cl, np_output_cf) + def testFunctionalFlatten(self): x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32') y = core_layers.flatten(x, name='flatten') -- GitLab From 579aecd2de1f0582858f83e3c8da2a8dbb57993b Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Sun, 18 Mar 2018 20:08:59 +0000 Subject: [PATCH 0118/1085] added dtype to test --- tensorflow/python/layers/core_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index 0d019897aa..31f3a4e0b0 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -477,7 +477,7 @@ class FlattenTest(test.TestCase): self.assertEqual(shape.as_list(), [None, None]) def testDataFormat(self): - np_input_channels_last = np.arange(3, 7).reshape([1, 2, 3, 2]) + np_input_channels_last = np.arange(12, dtype='float32').reshape([1, 2, 3, 2]) with self.test_session() as sess: x = array_ops.placeholder(shape=(1, 2, 3, 2), dtype='float32') -- GitLab From 76964f315f7c52d63ce6578d87278a96c7394ece Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Sun, 18 Mar 2018 22:01:21 +0000 Subject: [PATCH 0119/1085] pylint compliance --- tensorflow/python/layers/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index 5f89e3c0c3..5919fa543e 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -268,14 +268,14 @@ def dropout(inputs, @tf_export('layers.Flatten') class Flatten(keras_layers.Flatten, base.Layer): """Flattens an input tensor while preserving the batch axis (axis 0). - + Arguments: data_format: A string, one of `channels_last` (default) or `channels_first`. The ordering of the dimensions in the inputs. `channels_last` corresponds to inputs with shape `(batch, ..., channels)` while `channels_first` corresponds to inputs with shape `(batch, channels, ...)`. - + Examples: ``` -- GitLab From 110baa57112a95c2644896ce6ff75894e1ae61c7 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Sun, 18 Mar 2018 23:10:55 +0000 Subject: [PATCH 0120/1085] Extended to N-dims --- tensorflow/python/layers/core_test.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index 31f3a4e0b0..d5b8a0ff65 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -476,15 +476,31 @@ class FlattenTest(test.TestCase): shape = core_layers.Flatten().compute_output_shape((None, 3, None)) self.assertEqual(shape.as_list(), [None, None]) - def testDataFormat(self): - np_input_channels_last = np.arange(12, dtype='float32').reshape([1, 2, 3, 2]) + def testDataFormat5d(self): + np_input_channels_last = np.arange(120, dtype='float32').reshape([1, 5, 4, 3, 2]) with self.test_session() as sess: - x = array_ops.placeholder(shape=(1, 2, 3, 2), dtype='float32') + x = array_ops.placeholder(shape=(1, 5, 4, 3, 2), dtype='float32') y = core_layers.Flatten(data_format='channels_last')(x) np_output_cl = sess.run(y, feed_dict={x: np_input_channels_last}) - x = array_ops.placeholder(shape=(1, 2, 3, 2), dtype='float32') + x = array_ops.placeholder(shape=(1, 2, 5, 4, 3), dtype='float32') + y = core_layers.Flatten(data_format='channels_first')(x) + np_input_channels_first = np.transpose(np_input_channels_last, + [0, 4, 1, 2, 3]) + np_output_cf = sess.run(y, feed_dict={x: np_input_channels_first}) + + self.assertEqual(np_output_cl, np_output_cf) + + def testDataFormat4d(self): + np_input_channels_last = np.arange(24, dtype='float32').reshape([1, 4, 3, 2]) + + with self.test_session() as sess: + x = array_ops.placeholder(shape=(1, 4, 3, 2), dtype='float32') + y = core_layers.Flatten(data_format='channels_last')(x) + np_output_cl = sess.run(y, feed_dict={x: np_input_channels_last}) + + x = array_ops.placeholder(shape=(1, 2, 4, 3), dtype='float32') y = core_layers.Flatten(data_format='channels_first')(x) np_input_channels_first = np.transpose(np_input_channels_last, [0, 3, 1, 2]) -- GitLab From 4de591a03a9bd49a05d67fe48f9358dbdac51561 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Sat, 30 Jun 2018 08:14:40 +0100 Subject: [PATCH 0121/1085] Fixed Pylint Issues --- tensorflow/python/layers/core_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index d5b8a0ff65..8ad0e8c4ba 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -477,7 +477,8 @@ class FlattenTest(test.TestCase): self.assertEqual(shape.as_list(), [None, None]) def testDataFormat5d(self): - np_input_channels_last = np.arange(120, dtype='float32').reshape([1, 5, 4, 3, 2]) + np_input_channels_last = np.arange(120, dtype='float32').reshape( + [1, 5, 4, 3, 2]) with self.test_session() as sess: x = array_ops.placeholder(shape=(1, 5, 4, 3, 2), dtype='float32') @@ -493,7 +494,8 @@ class FlattenTest(test.TestCase): self.assertEqual(np_output_cl, np_output_cf) def testDataFormat4d(self): - np_input_channels_last = np.arange(24, dtype='float32').reshape([1, 4, 3, 2]) + np_input_channels_last = np.arange(24, dtype='float32').reshape( + [1, 4, 3, 2]) with self.test_session() as sess: x = array_ops.placeholder(shape=(1, 4, 3, 2), dtype='float32') -- GitLab From 46fc7a9530e9c8f6bf909de8df8c97e4b38a99a5 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Mon, 23 Jul 2018 23:06:48 +0100 Subject: [PATCH 0122/1085] Fixed Tests --- tensorflow/python/layers/core_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index 8ad0e8c4ba..22ed75dda7 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -491,7 +491,7 @@ class FlattenTest(test.TestCase): [0, 4, 1, 2, 3]) np_output_cf = sess.run(y, feed_dict={x: np_input_channels_first}) - self.assertEqual(np_output_cl, np_output_cf) + self.assertAllEqual(np_output_cl, np_output_cf) def testDataFormat4d(self): np_input_channels_last = np.arange(24, dtype='float32').reshape( @@ -508,7 +508,7 @@ class FlattenTest(test.TestCase): [0, 3, 1, 2]) np_output_cf = sess.run(y, feed_dict={x: np_input_channels_first}) - self.assertEqual(np_output_cl, np_output_cf) + self.assertAllEqual(np_output_cl, np_output_cf) def testFunctionalFlatten(self): x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32') -- GitLab From da930ea7fd16c903346ff36f5f57548dbea98bdc Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Tue, 21 Aug 2018 08:17:29 +0100 Subject: [PATCH 0123/1085] Updated golden --- tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 509ceff9df..e65ffeb12e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -832,10 +832,6 @@ tf_module { name: "broadcast_static_shape" argspec: "args=[\'shape_x\', \'shape_y\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "broadcast_to" - argspec: "args=[\'input\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "case" argspec: "args=[\'pred_fn_pairs\', \'default\', \'exclusive\', \'strict\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'False\', \'case\'], " -- GitLab From 459accb2b7bdea542415f3a744cbe9e348f847d6 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Tue, 21 Aug 2018 21:02:13 +0100 Subject: [PATCH 0124/1085] Updated layers --- tensorflow/tools/api/golden/v1/tensorflow.layers.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.pbtxt index df74c32e1f..5d9ea2e5a3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.pbtxt @@ -122,7 +122,7 @@ tf_module { } member_method { name: "flatten" - argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'inputs\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'channels_last\', \'None\'], " } member_method { name: "max_pooling1d" -- GitLab From a58135a6a9637db0908c88f39df22b69bafaec3d Mon Sep 17 00:00:00 2001 From: Joe Yearsley Date: Sat, 25 Aug 2018 16:04:34 +0100 Subject: [PATCH 0125/1085] Updated protobuf --- tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index e65ffeb12e..509ceff9df 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -832,6 +832,10 @@ tf_module { name: "broadcast_static_shape" argspec: "args=[\'shape_x\', \'shape_y\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "broadcast_to" + argspec: "args=[\'input\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "case" argspec: "args=[\'pred_fn_pairs\', \'default\', \'exclusive\', \'strict\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'False\', \'case\'], " -- GitLab From 8e87c649fc290c758c4240bf202de0c7f0f3a4ad Mon Sep 17 00:00:00 2001 From: Joe Yearsley Date: Sat, 29 Sep 2018 17:38:44 +0100 Subject: [PATCH 0126/1085] Updated v2 --- tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt index df74c32e1f..5fd6ba1192 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt @@ -122,8 +122,8 @@ tf_module { } member_method { name: "flatten" - argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } + argspec: "args=[\'inputs\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'channels_last\', \'None\'], " +} member_method { name: "max_pooling1d" argspec: "args=[\'inputs\', \'pool_size\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'valid\', \'channels_last\', \'None\'], " -- GitLab From 32059ed204ecbee7828057d23a1c1daf561c87fd Mon Sep 17 00:00:00 2001 From: Joe Yearsley Date: Sat, 29 Sep 2018 17:42:52 +0100 Subject: [PATCH 0127/1085] Update tensorflow.layers.pbtxt --- tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt index 5fd6ba1192..5d9ea2e5a3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.layers.pbtxt @@ -122,8 +122,8 @@ tf_module { } member_method { name: "flatten" - argspec: "args=[\'inputs\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'channels_last\', \'None\'], " -} + argspec: "args=[\'inputs\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'channels_last\', \'None\'], " + } member_method { name: "max_pooling1d" argspec: "args=[\'inputs\', \'pool_size\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'valid\', \'channels_last\', \'None\'], " -- GitLab From 914f68bfc0d7629496cd5ef6a6104efc94b6eecc Mon Sep 17 00:00:00 2001 From: Balint Cristian Date: Sat, 29 Sep 2018 20:13:26 +0300 Subject: [PATCH 0128/1085] Add abseil_cpp cmake dependence. --- tensorflow/contrib/cmake/CMakeLists.txt | 6 +- .../contrib/cmake/external/abseil_cpp.cmake | 100 ++++++++++++++++++ .../contrib/cmake/modules/FindAbseilCpp.cmake | 72 +++++++++++++ 3 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/cmake/external/abseil_cpp.cmake create mode 100644 tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index c6d6f04168..dc0d2569c5 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -90,10 +90,12 @@ if (NOT WIN32) # Options for linking other libraries option(systemlib_ZLIB "Use the system installed library as shared objects instead of downloading ZLIB and statically linking to it: ZLIB" OFF) + option(systemlib_ABSEIL_CPP "Use the system installed library as shared objects instead of downloading ABSEIL_CPP and statically linking to it: ABSEIL_CPP" OFF) option(systemlib_ALL "Turn on every possible systemlib_* options" OFF) if (systemlib_ALL) set (systemlib_ZLIB ON) + set (systemlib_ABSEIL_CPP ON) endif (systemlib_ALL) endif() @@ -115,7 +117,7 @@ function(SHOW_VARIABLES) endfunction() # External dependencies -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) +set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external ${PROJECT_SOURCE_DIR}/modules) # Location where external projects will be downloaded set (DOWNLOAD_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/downloads" @@ -240,6 +242,7 @@ include(re2) include(cub) include(sqlite) include(double_conversion) +include(abseil_cpp) if (tensorflow_BUILD_CC_TESTS) include(googletest) endif() @@ -248,6 +251,7 @@ add_definitions(${ADD_CFLAGS}) link_directories(${ADD_LINK_DIRECTORY}) set(tensorflow_EXTERNAL_LIBRARIES + ${tensorflow_EXTERNAL_LIBRARIES} ${gif_STATIC_LIBRARIES} ${png_STATIC_LIBRARIES} ${jpeg_STATIC_LIBRARIES} diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake new file mode 100644 index 0000000000..c6c5021f60 --- /dev/null +++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake @@ -0,0 +1,100 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +if (systemlib_ABSEIL_CPP) + + find_package(AbseilCpp REQUIRED + absl_base + absl_spinlock_wait + absl_dynamic_annotations + absl_malloc_internal + absl_throw_delegate + absl_strings + str_format_internal + absl_bad_optional_access) + + include_directories(${ABSEIL_CPP_INCLUDE_DIR}) + list(APPEND tensorflow_EXTERNAL_LIBRARIES ${ABSEIL_CPP_LIBRARIES}) + + message(STATUS " abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}") + message(STATUS " abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}") + + add_custom_target(abseil_cpp_build) + list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) + +else (systemlib_ABSEIL_CPP) + + include (ExternalProject) + + set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) + set(abseil_cpp_URL https://github.com/abseil/abseil-cpp/archive/e01d95528ea2137a4a27a88d1f57c6cb260aafed.tar.gz) + set(abseil_cpp_HASH SHA256=84043ed402d2a2a6ba4cdddb7e85118b1158fd81fe4ac3a14adc343d054c1e2e) + set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) + + if(WIN32) + if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") + set(abseil_cpp_STATIC_LIBRARIES + ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib + ${abseil_cpp_BUILD}/absl/base/Release/absl_spinlock_wait.lib + ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib + ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib + ${abseil_cpp_BUILD}/absl/base/Release/absl_throw_delegate.lib + ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib + ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib + ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib) + else() + set(abseil_cpp_STATIC_LIBRARIES + ${abseil_cpp_BUILD}/absl/base/absl_base.lib + ${abseil_cpp_BUILD}/absl/base/absl_spinlock_wait.lib + ${abseil_cpp_BUILD}/absl/base/absl_dynamic_annotations.lib + ${abseil_cpp_BUILD}/absl/base/absl_malloc_internal.lib + ${abseil_cpp_BUILD}/absl/base/absl_throw_delegate.lib + ${abseil_cpp_BUILD}/absl/strings/absl_strings.lib + ${abseil_cpp_BUILD}/absl/strings/str_format_internal.lib + ${abseil_cpp_BUILD}/absl/types/absl_bad_optional_access.lib) + endif() + else() + set(abseil_cpp_STATIC_LIBRARIES + ${abseil_cpp_BUILD}/absl/base/libabsl_base.a + ${abseil_cpp_BUILD}/absl/base/libabsl_spinlock_wait.a + ${abseil_cpp_BUILD}/absl/base/libabsl_dynamic_annotations.a + ${abseil_cpp_BUILD}/absl/base/libabsl_malloc_internal.a + ${abseil_cpp_BUILD}/absl/base/libabsl_throw_delegate.a + ${abseil_cpp_BUILD}/absl/strings/libabsl_strings.a + ${abseil_cpp_BUILD}/absl/strings/libstr_format_internal.a + ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a) + endif() + + ExternalProject_Add(abseil_cpp_build + PREFIX abseil_cpp + URL ${abseil_cpp_URL} + URL_HASH ${abseil_cpp_HASH} + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES} + BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release + COMMAND ${CMAKE_COMMAND} --build . --config Release + INSTALL_COMMAND "" + CMAKE_CACHE_ARGS + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF + ) + + include_directories(${abseil_cpp_INCLUDE_DIR}) + list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES}) + + list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) + +endif (systemlib_ABSEIL_CPP) diff --git a/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake b/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake new file mode 100644 index 0000000000..d4f8bb1bec --- /dev/null +++ b/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake @@ -0,0 +1,72 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +find_path(ABSEIL_CPP_INCLUDE_DIR absl/base/config.h + HINTS "${ABSEIL_CPP_INCLUDE_DIR_HINTS}" + PATHS "$ENV{PROGRAMFILES}" + "$ENV{PROGRAMW6432}" + PATH_SUFFIXES "") + +if(EXISTS "${ABSEIL_CPP_INCLUDE_DIR}" AND NOT "${ABSEIL_CPP_INCLUDE_DIR}" STREQUAL "") + + if(NOT AbseilCpp_FIND_COMPONENTS) + # search all libraries if no COMPONENTS was requested + set(AbseilCpp_FIND_COMPONENTS + "absl_algorithm;absl_any;absl_bad_any_cast" + "absl_bad_optional_access;absl_base absl_container;absl_debugging" + "absl_dynamic_annotations;absl_examine_stack;absl_failure_signal_handler" + "absl_int128;absl_leak_check;absl_malloc_internal;absl_memory;absl_meta" + "absl_numeric;absl_optional;absl_span;absl_spinlock_wait;absl_stack_consumption" + "absl_stacktrace;absl_str_format;absl_strings;absl_symbolize;absl_synchronization" + "absl_throw_delegate;absl_time;absl_utility;str_format_extension_internal" + "str_format_internal;test_instance_tracker_lib") + endif() + + foreach(LIBNAME ${AbseilCpp_FIND_COMPONENTS}) + + unset(ABSEIL_CPP_LIBRARY CACHE) + + find_library(ABSEIL_CPP_LIBRARY + NAMES ${LIBNAME} + HINTS ${ABSEIL_CPP_LIBRARIES_DIR_HINTS}) + + if(ABSEIL_CPP_LIBRARY) + list(APPEND ABSEIL_CPP_LIBRARIES ${ABSEIL_CPP_LIBRARY}) + else() + message(FATAL_ERROR "\n" + "abseil_cpp library \"${LIBNAME}\" not found in system path.\n" + "Please provide locations using: -DABSEIL_CPP_LIBRARIES_DIR_HINTS:STRING=\"PATH\"\n") + endif() + + endforeach() + + unset(LIBNAME CACHE) + unset(ABSEIL_CPP_LIBRARY CACHE) + + set(ABSEIL_CPP_FOUND TRUE) + message(STATUS "Found abseil_cpp libraries") + + set(ABSEIL_CPP_INCLUDE_DIR "${ABSEIL_CPP_INCLUDE_DIR}" CACHE PATH "" FORCE) + mark_as_advanced(ABSEIL_CPP_INCLUDE_DIR) + + set(ABSEIL_CPP_LIBRARIES "${ABSEIL_CPP_LIBRARIES}" CACHE PATH "" FORCE) + mark_as_advanced(ABSEIL_CPP_LIBRARIES) + +else() + + message(FATAL_ERROR "\n" + "abseil_cpp headers not found in system path.\n" + "Please provide locations using: -DABSEIL_CPP_INCLUDE_DIR_HINTS:STRING=\"PATH\"\n") + +endif() -- GitLab From 70a395f9795a48c21bc35cdf1dc44778f73a7bba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 29 Sep 2018 11:58:55 -0700 Subject: [PATCH 0129/1085] Automated rollback of commit d78595d333c9b5c8a0705ba6852c08b107d6c462 PiperOrigin-RevId: 215073584 --- tensorflow/python/data/kernel_tests/BUILD | 1 - tensorflow/tensorflow.bzl | 39 ++++++++----------- .../tools/pip_package/pip_smoke_test.py | 2 +- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 99d7f70513..cadfe7f9e0 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -318,7 +318,6 @@ cuda_py_test( "//tensorflow/python:framework_test_lib", ], tags = [ - "no_oss", # TODO(b/116813115): Investigate timeout and re-enable. "no_windows_gpu", ], ) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index dead44c57e..cad5de1b0c 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -1798,29 +1798,22 @@ def cuda_py_test( flaky = 0, xla_enabled = False, grpc_enabled = False): - if main == None: - main = name + ".py" - for config in ["cpu", "gpu"]: - test_name = name - test_tags = tags - if config == "gpu": - test_name += "_gpu" - test_tags = test_tags + tf_cuda_tests_tags() - tf_py_test( - name = test_name, - size = size, - srcs = srcs, - data = data, - main = main, - args = args, - tags = test_tags, - shard_count = shard_count, - additional_deps = additional_deps, - kernels = kernels, - flaky = flaky, - xla_enabled = xla_enabled, - grpc_enabled = grpc_enabled, - ) + test_tags = tags + tf_cuda_tests_tags() + tf_py_test( + name = name, + size = size, + srcs = srcs, + data = data, + main = main, + args = args, + tags = test_tags, + shard_count = shard_count, + additional_deps = additional_deps, + kernels = kernels, + flaky = flaky, + xla_enabled = xla_enabled, + grpc_enabled = grpc_enabled, + ) register_extension_info( extension_name = "cuda_py_test", diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py index e7f9628fa6..c6ef82ccdc 100644 --- a/tensorflow/tools/pip_package/pip_smoke_test.py +++ b/tensorflow/tools/pip_package/pip_smoke_test.py @@ -142,7 +142,7 @@ def main(): missing_dependencies = [] # File extensions and endings to ignore - ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"] + ignore_extensions = ["_test", "_test.py"] ignored_files = 0 blacklisted_files = len(BLACKLIST) -- GitLab From 639d0dd8c1ba8d2956ccb59604c157de7ba0a7f2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 29 Sep 2018 12:00:53 -0700 Subject: [PATCH 0130/1085] Cleanup PiperOrigin-RevId: 215073641 --- tensorflow/core/BUILD | 3 --- tensorflow/core/profiler/BUILD | 1 - 2 files changed, 4 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 7da4b9fbd0..57819cec70 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -239,7 +239,6 @@ tf_proto_library( srcs = [], cc_api_version = 2, default_header = True, - java_api_version = 2, js_api_version = 2, protodeps = [ ":protos_all_proto", @@ -2385,7 +2384,6 @@ tf_proto_library( srcs = ERROR_CODES_PROTO_SRCS, cc_api_version = 2, default_header = True, - java_api_version = 2, js_api_version = 2, provide_cc_alias = True, ) @@ -2406,7 +2404,6 @@ tf_proto_library( srcs = COMMON_PROTO_SRCS + ADDITIONAL_CORE_PROTO_SRCS, cc_api_version = 2, default_header = True, - java_api_version = 2, js_api_version = 2, protodeps = [ ":error_codes_proto", diff --git a/tensorflow/core/profiler/BUILD b/tensorflow/core/profiler/BUILD index af034bdd7d..2bf371276e 100644 --- a/tensorflow/core/profiler/BUILD +++ b/tensorflow/core/profiler/BUILD @@ -40,7 +40,6 @@ tf_proto_library( name = "protos_all", srcs = glob(["**/*.proto"]), cc_api_version = 2, - java_api_version = 2, protodeps = tf_additional_all_protos(), visibility = ["//visibility:public"], ) -- GitLab From 4cf1b45b2e9188086bcb7d12654cd3e130e9b823 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 29 Sep 2018 14:13:01 -0700 Subject: [PATCH 0131/1085] Disable PinToHostOptimizer for NoOp. PiperOrigin-RevId: 215079134 --- .../core/grappler/optimizers/pin_to_host_optimizer.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.cc b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.cc index 2190d38937..89eb76046e 100644 --- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.cc @@ -169,7 +169,13 @@ bool IsTPUGraphDef(const GraphDef& def) { } // All the nodes that should be blacklisted and not swapped. -bool IsBlacklisted(const NodeDef& node) { return IsCollective(node); } +bool IsBlacklisted(const NodeDef& node) { + return + // Collective ops should not be swapped. + IsCollective(node) || + // NoOp breaks perf regression tests (probably due to group dependencies). + IsNoOp(node); +} } // end namespace internal Status PinToHostOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, -- GitLab From 2538e68a69e585696175bd972cae119e06bde294 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 29 Sep 2018 16:13:51 -0700 Subject: [PATCH 0132/1085] Remove workaround for symlinked headers. PiperOrigin-RevId: 215083669 --- third_party/gpus/cuda_configure.bzl | 33 +++++++++-------------------- third_party/py/python_configure.bzl | 4 ++-- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl index f5fdd3a75e..69f4599c16 100644 --- a/third_party/gpus/cuda_configure.bzl +++ b/third_party/gpus/cuda_configure.bzl @@ -1107,8 +1107,8 @@ def symlink_genrule_for_dir( # $(@D) will include the full path to the file. dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i] - # On Windows, symlink is not supported, so we just copy all the files. - cmd = "cp -f" if _is_windows(repository_ctx) else "ln -s" + # Copy the headers to create a sandboxable setup. + cmd = "cp -f" command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) outs.append(' "' + dest_dir + dest_files[i] + '",') genrule = _genrule( @@ -1334,27 +1334,14 @@ def _create_local_cuda_repository(repository_ctx): cuda_defines["%{host_compiler_path}"] = "clang/bin/crosstool_wrapper_driver_is_not_gcc" cuda_defines["%{host_compiler_warnings}"] = "" - # TODO(klimek): We currently need to inject "/" as builtin directory path - # to disable bazel's dependency checks. - # The problem is that: - # - the python rules symlink the python headers into the bazel root - # - the rules use 'includes' in the BUILD file to redirect includes of the - # python headers through those paths - # - bazel currently uses -isystem for include paths specified via 'includes' - # - gcc follows symlinks when resolving files via -isystem paths, and puts - # the resolved paths into the .d file, which makes the dependency check - # fail for bazel - # There are multiple possible ways to solve this: - # 1. make bazel not use -isystem for paths specified via 'includes' - # 2. cp the headers instead of symlinking them - # - # Once this is fixed, the right builtin directory path is: - # (host_compiler_includes + - # "\n cxx_builtin_include_directory: \"%s\"" % cuda_include_path) - # The cuda directory needs to be passed, as there is currently no rule - # providing the cuda headers in the same way the python headers are - # provided. - cuda_defines["%{host_compiler_includes}"] = "\n cxx_builtin_include_directory: \"/\"" + # nvcc has the system include paths built in and will automatically + # search them; we cannot work around that, so we add the relevant cuda + # system paths to the allowed compiler specific include paths. + cuda_defines["%{host_compiler_includes}"] = ( + host_compiler_includes + "\n" + + _cuda_include_path(repository_ctx, cuda_config) + + "\n cxx_builtin_include_directory: \"%s\"" % cupti_header_dir + + "\n cxx_builtin_include_directory: \"%s\"" % cudnn_header_dir) nvcc_path = str(repository_ctx.path("%s/bin/nvcc%s" % ( cuda_config.cuda_toolkit_path, diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 3c7e5c8469..53264630a1 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -130,8 +130,8 @@ def _symlink_genrule_for_dir(repository_ctx, src_dir, dest_dir, genrule_name, # If we have only one file to link we do not want to use the dest_dir, as # $(@D) will include the full path to the file. dest = '$(@D)/' + dest_dir + dest_files[i] if len(dest_files) != 1 else '$(@D)/' + dest_files[i] - # On Windows, symlink is not supported, so we just copy all the files. - cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' + # Copy the headers to create a sandboxable setup. + cmd = 'cp -f' command.append(cmd + ' "%s" "%s"' % (src_files[i] , dest)) outs.append(' "' + dest_dir + dest_files[i] + '",') genrule = _genrule(src_dir, genrule_name, " && ".join(command), -- GitLab From e0da6256cd116d17057374594f2fc191cf201f42 Mon Sep 17 00:00:00 2001 From: AG Ramesh Date: Sat, 29 Sep 2018 23:29:28 -0700 Subject: [PATCH 0133/1085] Fixed format errors reported by clang-format --- tensorflow/core/common_runtime/process_util.cc | 11 ++++++----- tensorflow/core/common_runtime/threadpool_device.cc | 6 +++--- tensorflow/core/util/util.cc | 8 ++------ tensorflow/core/util/util.h | 2 +- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index b3064a4c08..4570496637 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -62,15 +62,16 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) { // Set inter_op conservatively to avoid thread oversubscription that could // lead to severe perf degradations and OMP resource exhaustion int mkl_intra_op = 1; - #ifdef _OPENMP +#ifdef _OPENMP mkl_intra_op = omp_get_max_threads(); - #endif // _OPENMP +#endif // _OPENMP CHECK_GE(mkl_intra_op, 1); const int32 mkl_inter_op = std::max( (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2); - VLOG(0) << "Creating new thread pool with default inter op setting: " - << mkl_inter_op - << ". Tune using inter_op_parallelism_threads for best performance."; + VLOG(0) + << "Creating new thread pool with default inter op setting: " + << mkl_inter_op + << ". Tune using inter_op_parallelism_threads for best performance."; return mkl_inter_op; } #endif // INTEL_MKL diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index f188016610..6404d8bc6a 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -51,8 +51,7 @@ ThreadPoolDevice::ThreadPoolDevice(const SessionOptions& options, scoped_allocator_mgr_(new ScopedAllocatorMgr(name)) { #ifdef INTEL_MKL // Early return when MKL is disabled - if (DisableMKL()) - return; + if (DisableMKL()) return; #ifdef _OPENMP const char* user_omp_threads = getenv("OMP_NUM_THREADS"); if (user_omp_threads == nullptr) { @@ -118,7 +117,8 @@ class MklCPUAllocatorFactory : public AllocatorFactory { }; #ifdef ENABLE_MKL -REGISTER_MEM_ALLOCATOR("MklCPUAllocator", (DisableMKL() ? 50 : 200), MklCPUAllocatorFactory); +REGISTER_MEM_ALLOCATOR("MklCPUAllocator", (DisableMKL() ? 50 : 200), + MklCPUAllocatorFactory); #endif // ENABLE_MKL } // namespace diff --git a/tensorflow/core/util/util.cc b/tensorflow/core/util/util.cc index 44d5becb9c..489999d1e8 100644 --- a/tensorflow/core/util/util.cc +++ b/tensorflow/core/util/util.cc @@ -122,11 +122,7 @@ string SliceDebugString(const TensorShape& shape, const int64 flat) { #ifdef INTEL_MKL bool DisableMKL() { - enum MklStatus { - MKL_DEFAULT = 0, - MKL_ON = 1, - MKL_OFF = 2 - }; + enum MklStatus { MKL_DEFAULT = 0, MKL_ON = 1, MKL_OFF = 2 }; static MklStatus status = MKL_DEFAULT; if (status == MKL_DEFAULT) { char* tf_disable_mkl = getenv("TF_DISABLE_MKL"); @@ -139,5 +135,5 @@ bool DisableMKL() { } return status == MKL_OFF ? true : false; } -#endif +#endif // INTEL_MKL } // namespace tensorflow diff --git a/tensorflow/core/util/util.h b/tensorflow/core/util/util.h index ba90ad52c2..4aa47aa48a 100644 --- a/tensorflow/core/util/util.h +++ b/tensorflow/core/util/util.h @@ -59,7 +59,7 @@ string SliceDebugString(const TensorShape& shape, const int64 flat); // disable MKL in runtime #ifdef INTEL_MKL bool DisableMKL(); -#endif +#endif // INTEL_MKL } // namespace tensorflow -- GitLab From 2b456a2b5dc6b5bb092b3986a400acb77b21a30f Mon Sep 17 00:00:00 2001 From: "Xiaoming (Jason) Cui" Date: Sun, 30 Sep 2018 01:12:34 -0700 Subject: [PATCH 0134/1085] Added some minor format changes --- tensorflow/core/common_runtime/process_util.cc | 6 +++--- tensorflow/core/common_runtime/threadpool_device.cc | 6 +++--- tensorflow/core/util/util.cc | 2 +- tensorflow/core/util/util.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index b3064a4c08..c75d8a8ce6 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -62,15 +62,15 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) { // Set inter_op conservatively to avoid thread oversubscription that could // lead to severe perf degradations and OMP resource exhaustion int mkl_intra_op = 1; - #ifdef _OPENMP +#ifdef _OPENMP mkl_intra_op = omp_get_max_threads(); - #endif // _OPENMP +#endif // _OPENMP CHECK_GE(mkl_intra_op, 1); const int32 mkl_inter_op = std::max( (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2); VLOG(0) << "Creating new thread pool with default inter op setting: " << mkl_inter_op - << ". Tune using inter_op_parallelism_threads for best performance."; + << ".Tune using inter_op_parallelism_threads for best performance."; return mkl_inter_op; } #endif // INTEL_MKL diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index f188016610..6404d8bc6a 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -51,8 +51,7 @@ ThreadPoolDevice::ThreadPoolDevice(const SessionOptions& options, scoped_allocator_mgr_(new ScopedAllocatorMgr(name)) { #ifdef INTEL_MKL // Early return when MKL is disabled - if (DisableMKL()) - return; + if (DisableMKL()) return; #ifdef _OPENMP const char* user_omp_threads = getenv("OMP_NUM_THREADS"); if (user_omp_threads == nullptr) { @@ -118,7 +117,8 @@ class MklCPUAllocatorFactory : public AllocatorFactory { }; #ifdef ENABLE_MKL -REGISTER_MEM_ALLOCATOR("MklCPUAllocator", (DisableMKL() ? 50 : 200), MklCPUAllocatorFactory); +REGISTER_MEM_ALLOCATOR("MklCPUAllocator", (DisableMKL() ? 50 : 200), + MklCPUAllocatorFactory); #endif // ENABLE_MKL } // namespace diff --git a/tensorflow/core/util/util.cc b/tensorflow/core/util/util.cc index 44d5becb9c..6e78777dd9 100644 --- a/tensorflow/core/util/util.cc +++ b/tensorflow/core/util/util.cc @@ -139,5 +139,5 @@ bool DisableMKL() { } return status == MKL_OFF ? true : false; } -#endif +#endif // INTEL_MKL } // namespace tensorflow diff --git a/tensorflow/core/util/util.h b/tensorflow/core/util/util.h index ba90ad52c2..4aa47aa48a 100644 --- a/tensorflow/core/util/util.h +++ b/tensorflow/core/util/util.h @@ -59,7 +59,7 @@ string SliceDebugString(const TensorShape& shape, const int64 flat); // disable MKL in runtime #ifdef INTEL_MKL bool DisableMKL(); -#endif +#endif // INTEL_MKL } // namespace tensorflow -- GitLab From a00fe72261cf6fe4a00467139e401de14c16224c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 30 Sep 2018 02:00:58 -0700 Subject: [PATCH 0135/1085] compat: Update forward compatibility horizon to 2018-09-30 PiperOrigin-RevId: 215109054 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 24a795c787..1f7cfe48b3 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -26,7 +26,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 9, 29) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 9, 30) @tf_export("compat.forward_compatible") -- GitLab From 4ecce5aa64587afe1cd07ee4c92bbb5ce2cf85df Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Sun, 30 Sep 2018 06:52:22 -0700 Subject: [PATCH 0136/1085] Removing the setuptools upper limit. PiperOrigin-RevId: 215120867 --- tensorflow/tools/pip_package/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index b95e1f5c87..a9d8b0cff5 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -56,7 +56,6 @@ REQUIRED_PACKAGES = [ 'numpy >= 1.13.3', 'six >= 1.10.0', 'protobuf >= 3.6.0', - 'setuptools <= 39.1.0', 'tensorboard >= 1.11.0, < 1.12.0', 'termcolor >= 1.1.0', ] -- GitLab From 5fa4e1ac928b0512b28e955c588c5a7eab2ea046 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 30 Sep 2018 11:57:45 -0700 Subject: [PATCH 0137/1085] Parallel_for: fix converters for some ops that don't support broadcasting. PiperOrigin-RevId: 215133508 --- tensorflow/python/ops/parallel_for/pfor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py index e0f6d51881..83cbe64ff2 100644 --- a/tensorflow/python/ops/parallel_for/pfor.py +++ b/tensorflow/python/ops/parallel_for/pfor.py @@ -1987,14 +1987,12 @@ def _convert_cast(pfor_input): @RegisterPForWithArgs("Pow", math_ops.pow) @RegisterPForWithArgs("RealDiv", math_ops.divide) @RegisterPForWithArgs("Real", math_ops.real) -@RegisterPForWithArgs("ReciprocalGrad", math_ops.reciprocal_grad) @RegisterPForWithArgs("Reciprocal", math_ops.reciprocal) @RegisterPForWithArgs("Relu6", nn_ops.relu6) @RegisterPForWithArgs("Relu", nn_ops.relu) @RegisterPForWithArgs("RightShift", bitwise_ops.right_shift) @RegisterPForWithArgs("Rint", math_ops.rint) @RegisterPForWithArgs("Round", math_ops.round) -@RegisterPForWithArgs("RsqrtGrad", math_ops.rsqrt_grad) @RegisterPForWithArgs("Rsqrt", math_ops.rsqrt) @RegisterPForWithArgs("Selu", nn_ops.selu) @RegisterPForWithArgs("Sigmoid", math_ops.sigmoid) @@ -2003,7 +2001,6 @@ def _convert_cast(pfor_input): @RegisterPForWithArgs("Sin", math_ops.sin) @RegisterPForWithArgs("Softplus", nn_ops.softplus) @RegisterPForWithArgs("Softsign", nn_ops.softsign) -@RegisterPForWithArgs("SqrtGrad", math_ops.sqrt_grad) @RegisterPForWithArgs("Sqrt", math_ops.sqrt) @RegisterPForWithArgs("SquaredDifference", math_ops.squared_difference) @RegisterPForWithArgs("Square", math_ops.square) @@ -2095,6 +2092,9 @@ def _convert_biasaddgrad(pfor_input): @RegisterPForWithArgs("SoftplusGrad") @RegisterPForWithArgs("SoftsignGrad") @RegisterPForWithArgs("TanhGrad") +@RegisterPForWithArgs("SqrtGrad") +@RegisterPForWithArgs("RsqrtGrad") +@RegisterPForWithArgs("ReciprocalGrad") def _convert_grads(pfor_input, op_type, *args, **kw_args): del args del kw_args -- GitLab From 76c4853b50f201b4a809ac66746c798e049b294c Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Sun, 30 Sep 2018 20:03:29 -0700 Subject: [PATCH 0138/1085] Bump the version of protobuf TF pip package depends on. Fixes #21719 PiperOrigin-RevId: 215154273 --- tensorflow/tools/pip_package/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index a9d8b0cff5..88c9c20d36 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -55,7 +55,7 @@ REQUIRED_PACKAGES = [ 'keras_preprocessing >= 1.0.3', 'numpy >= 1.13.3', 'six >= 1.10.0', - 'protobuf >= 3.6.0', + 'protobuf >= 3.6.1', 'tensorboard >= 1.11.0, < 1.12.0', 'termcolor >= 1.1.0', ] -- GitLab From b797bfb750504e03a38a988c44e3c52e902e87c4 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Sun, 30 Sep 2018 22:34:28 -0700 Subject: [PATCH 0139/1085] [HloOrdering] Make parameter always defined before other instructions. - Make parameter always defined before other instructions. - Add extra indentations to the predecessor field in ToString() method to make it clear. PiperOrigin-RevId: 215162840 --- .../compiler/xla/service/hlo_ordering.cc | 10 +++++++--- .../compiler/xla/service/hlo_ordering_test.cc | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc index f1dc08bafa..23d41d91d6 100644 --- a/tensorflow/compiler/xla/service/hlo_ordering.cc +++ b/tensorflow/compiler/xla/service/hlo_ordering.cc @@ -92,14 +92,18 @@ bool HloOrdering::ExecutesBefore(const HloInstruction* a, } bool HloOrdering::IsDefinedBefore(const HloValue& a, const HloValue& b) const { - // If 'b' is an entry param then 'a' cannot be defined before 'b' because 'b' - // is live into the module. + // Entry parameter should always be defined before other instructions. const HloModule* module = b.defining_instruction()->parent()->parent(); if (b.defining_instruction()->parent() == module->entry_computation() && b.defining_instruction()->opcode() == HloOpcode::kParameter) { return false; } + if (a.defining_instruction()->parent() == module->entry_computation() && + a.defining_instruction()->opcode() == HloOpcode::kParameter) { + return true; + } + // Phi values require special handling. Because XLA does not have a phi // instruction, the definition instruction of the phis values are // placeholders: either the subcomputation parameter (body or condition) or @@ -316,7 +320,7 @@ string PredecessorHloOrdering::ToStringHelper(const string& name) const { for (auto predecessor : all) { if (predecessors_.at(computation) ->IsReachable(predecessor, instruction)) { - pieces.push_back(absl::StrFormat(" %s", predecessor->name())); + pieces.push_back(absl::StrFormat(" %s", predecessor->name())); } } } diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc index 00970bcda3..b045adc964 100644 --- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc +++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc @@ -174,6 +174,26 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) { EXPECT_FALSE(ordering.ExecutesBefore(body_param, cond_param)); } +TEST_F(HloOrderingTest, ParametersDefinedBeforeOthers) { + // Entry parameter should always be defined before other instruction. + auto module = CreateNewModule(); + const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); + auto builder = HloComputation::Builder(TestName()); + auto constant = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0))); + auto param = builder.AddInstruction( + HloInstruction::CreateParameter(0, scalar_shape, "param")); + module->AddEntryComputation(builder.Build()); + TF_ASSERT_OK_AND_ASSIGN(auto dataflow, + HloDataflowAnalysis::Run(*module, /*ssa_form=*/true)); + + DependencyHloOrdering ordering(module.get()); + EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(param), + dataflow->GetValueDefinedAt(constant))); + EXPECT_TRUE(!ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(constant), + dataflow->GetValueDefinedAt(param))); +} + TEST_F(HloOrderingTest, ValuesInWhileComputations) { // Tests the ordering of values (defined by dataflow analysis) in the body and // condition of a while instruction. HLO code: -- GitLab From 03c5f9cdce62f6711b91fe81505e3c085e54a771 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 02:03:50 -0700 Subject: [PATCH 0140/1085] compat: Update forward compatibility horizon to 2018-10-01 PiperOrigin-RevId: 215179315 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 1f7cfe48b3..bea5aa990f 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -26,7 +26,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 9, 30) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 10, 1) @tf_export("compat.forward_compatible") -- GitLab From 0fd21d8c34e15bc3013e93014d101b672e1f3687 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 02:41:01 -0700 Subject: [PATCH 0141/1085] [TF:XLA] Teach deadness analysis more of distributive property. PiperOrigin-RevId: 215183847 --- tensorflow/compiler/jit/deadness_analysis.cc | 107 ++++++++++++++---- .../compiler/jit/deadness_analysis_test.cc | 31 ++++- 2 files changed, 112 insertions(+), 26 deletions(-) diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc index 9128b48da3..25e2e9a7af 100644 --- a/tensorflow/compiler/jit/deadness_analysis.cc +++ b/tensorflow/compiler/jit/deadness_analysis.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/compiler/jit/deadness_analysis.h" +#include "absl/algorithm/container.h" #include "absl/strings/str_join.h" #include "tensorflow/compiler/jit/deadness_analysis_internal.h" #include "tensorflow/core/graph/algorithm.h" @@ -383,6 +384,8 @@ class PredicateFactory { } Predicate* MakeAndOrImpl(absl::Span operands, bool is_and); + Predicate* MakeInternedAndOr(std::vector simplified_ops, + Predicate::Kind pred_kind); // Predicate instances are interned, meaning that there is only a single // instance of a Predicate object with a given content. This makes checking @@ -429,11 +432,40 @@ class PredicateFactory { interned_symbol_instances_; }; +Predicate* PredicateFactory::MakeInternedAndOr( + std::vector simplified_ops, Predicate::Kind pred_kind) { + std::stable_sort( + simplified_ops.begin(), simplified_ops.end(), + [](Predicate* a, Predicate* b) { return a->hash() < b->hash(); }); + + auto it = interned_and_or_instances_.find({pred_kind, simplified_ops}); + if (it != interned_and_or_instances_.end()) { + return it->second.get(); + } + + simplified_ops.shrink_to_fit(); + // NB! Because we'll use a non-owning reference to simplified_ops in the + // key for interned_and_or_instances_ we need to be careful to std::move() + // it all the way through. + absl::Span operands_slice = simplified_ops; + std::unique_ptr new_pred = + pred_kind == Predicate::Kind::kAnd + ? Make(std::move(simplified_ops)) + : Make(std::move(simplified_ops)); + + Predicate* new_pred_ptr = new_pred.get(); + interned_and_or_instances_.emplace( + SignatureForAndOr(pred_kind, operands_slice), std::move(new_pred)); + return new_pred_ptr; +} + // Common code to create AndPredicate or OrPredicate instances. Predicate* PredicateFactory::MakeAndOrImpl( absl::Span operands, bool is_and) { Predicate::Kind pred_kind = is_and ? Predicate::Kind::kAnd : Predicate::Kind::kOr; + Predicate::Kind other_pred_kind = + is_and ? Predicate::Kind::kOr : Predicate::Kind::kAnd; gtl::FlatSet simplified_ops_set; std::vector simplified_ops; for (Predicate* op : operands) { @@ -472,30 +504,63 @@ Predicate* PredicateFactory::MakeAndOrImpl( } } - std::stable_sort( - simplified_ops.begin(), simplified_ops.end(), - [](Predicate* a, Predicate* b) { return a->hash() < b->hash(); }); + // If all ops contain the same subop, then factor it out thanks to the + // distributive property. Such as: + // - (A & B) | (A & C) | (A & D) => A & (B | C | D) + // - (A | B) & (A | C) & (A | D) => A | (B & C & D) + // + // First find any predicates contained in all subops. + std::vector common_inner_operands; + gtl::FlatSet common_inner_operands_set; + for (Predicate* op : simplified_ops) { + if (op->kind() != other_pred_kind) { + common_inner_operands.clear(); + break; + } - auto it = interned_and_or_instances_.find({pred_kind, simplified_ops}); - if (it == interned_and_or_instances_.end()) { - simplified_ops.shrink_to_fit(); - // NB! Because we'll use a non-owning reference to simplified_ops in the - // key for interned_and_or_instances_ we need to be careful to std::move() - // it all the way through. - absl::Span operands_slice = simplified_ops; - std::unique_ptr new_pred = - is_and ? Make(std::move(simplified_ops)) - : Make(std::move(simplified_ops)); + if (common_inner_operands.empty()) { + common_inner_operands.insert(common_inner_operands.end(), + op->GetOperands().begin(), + op->GetOperands().end()); + } else { + std::vector sub_ops_intersection; + common_inner_operands.clear(); + absl::c_copy_if(op->GetOperands(), + std::back_inserter(common_inner_operands), + [&](Predicate* sub_op) { + return common_inner_operands_set.count(sub_op) == 1; + }); + } + if (common_inner_operands.empty()) break; + common_inner_operands_set.clear(); + common_inner_operands_set.insert(common_inner_operands.begin(), + common_inner_operands.end()); + } - Predicate* new_pred_ptr = new_pred.get(); - CHECK(interned_and_or_instances_ - .emplace(SignatureForAndOr(pred_kind, operands_slice), - std::move(new_pred)) - .second); - return new_pred_ptr; - } else { - return it->second.get(); + if (common_inner_operands.empty()) { + return MakeInternedAndOr(std::move(simplified_ops), pred_kind); } + + // For all predicates that can be factored out, remove them and recreate the + // subops. + std::vector factored_ops; + for (Predicate* op : simplified_ops) { + std::vector new_sub_op_ops; + absl::c_copy_if(op->GetOperands(), std::back_inserter(new_sub_op_ops), + [&](Predicate* sub_op) { + return std::find(common_inner_operands.begin(), + common_inner_operands.end(), + sub_op) == common_inner_operands.end(); + }); + factored_ops.push_back(MakeAndOrImpl(new_sub_op_ops, !is_and)); + } + + Predicate* new_inner_op = MakeAndOrImpl(factored_ops, is_and); + std::vector outer_ops; + outer_ops.push_back(new_inner_op); + outer_ops.insert(outer_ops.end(), common_inner_operands.begin(), + common_inner_operands.end()); + return MakeAndOrImpl(outer_ops, !is_and); } class DeadnessAnalysisImpl : public DeadnessAnalysis { diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc index 28a56044d5..617e31488c 100644 --- a/tensorflow/compiler/jit/deadness_analysis_test.cc +++ b/tensorflow/compiler/jit/deadness_analysis_test.cc @@ -384,10 +384,31 @@ TEST(DeadnessAnalysisTest, OrOfAnd) { EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add2.node())); } -TEST(DeadnessAnalysisTest, NEGATIVE_AndOrDistributive) { - // This demonstrates one of the weaknesses in the current approach -- since we - // only do some basic simplifications we can't see that "(A|B)&C" == - // "(A&C)|(B&C)". +TEST(DeadnessAnalysisTest, AndOrDistributiveSimplified) { + // (*A | (~*A & ((~*B & ~*A) | (~*A & *B)))) == #true + Scope root = Scope::NewRootScope().ExitOnError(); + + ops::Switch sw_0 = CreateSwitch(root, "A"); + ops::Switch sw_1 = CreateSwitch(root, "B"); + Output add0 = + ops::Add(root.WithOpName("and0"), sw_0.output_false, sw_1.output_true); + Output add1 = + ops::Add(root.WithOpName("and1"), sw_0.output_false, sw_1.output_false); + ops::Merge or2(root.WithOpName("or2"), {add0, add1}); + Output add3 = + ops::Add(root.WithOpName("and3"), or2.output, sw_0.output_false); + ops::Merge or4(root.WithOpName("or4"), {add3, sw_0.output_true}); + + std::unique_ptr result; + TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result)); + + PredicateMapTy predicate_map; + TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map)); + EXPECT_EQ(predicate_map[ControlOutputFor(or4.output)], "#true"); +} + +TEST(DeadnessAnalysisTest, AndOrDistributive) { + // (A|B)&C == (A&C)|(B&C) Scope root = Scope::NewRootScope().ExitOnError(); ops::Switch sw_0 = CreateSwitch(root, "0"); @@ -408,7 +429,7 @@ TEST(DeadnessAnalysisTest, NEGATIVE_AndOrDistributive) { std::unique_ptr result; TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result)); - EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add2.node())); + EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add3.node())); } TEST(DeadnessAnalysisTest, Ternary) { -- GitLab From c1c63c936c4bc51b401b82fbe54ed1945f49a314 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 03:27:05 -0700 Subject: [PATCH 0142/1085] Moves the creation of regularizer ops in get_variable out of surrounding context. This resembles the behaviour for initializer ops. PiperOrigin-RevId: 215187942 --- tensorflow/python/ops/variable_scope.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index af5c7d4050..5032ca79f9 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -939,7 +939,8 @@ class _VariableStore(object): if regularizer: with ops.colocate_with(v): with ops.name_scope(name + "/Regularizer/"): - loss = regularizer(v) + with ops.init_scope(): + loss = regularizer(v) if loss is not None: if context.executing_eagerly(): v_name = "v_%s" % type(v) -- GitLab From 9a169bf3ba840af8ab3caae7ea1c69c682be3ab7 Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Mon, 1 Oct 2018 03:34:35 -0700 Subject: [PATCH 0143/1085] Add allowed optimizations to GrapplerItem. (1) Skip UnaryOpComposition rewrite if the optimized graph needs to have a gradient registered for all nodes. PiperOrigin-RevId: 215188461 --- tensorflow/core/grappler/grappler_item.cc | 1 + tensorflow/core/grappler/grappler_item.h | 9 ++ tensorflow/core/grappler/op_types.cc | 4 + tensorflow/core/grappler/op_types.h | 1 + tensorflow/core/grappler/optimizers/BUILD | 2 + .../optimizers/arithmetic_optimizer.cc | 4 + .../grappler/optimizers/meta_optimizer.cc | 19 +++ .../optimizers/meta_optimizer_test.cc | 126 ++++++++++++++++++ 8 files changed, 166 insertions(+) diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc index bbc0fedd22..2c490f3966 100644 --- a/tensorflow/core/grappler/grappler_item.cc +++ b/tensorflow/core/grappler/grappler_item.cc @@ -38,6 +38,7 @@ GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef* graph_def) { restore_op = other.restore_op; save_restore_loc_tensor = other.save_restore_loc_tensor; queue_runners = other.queue_runners; + allowed_optimizations = other.allowed_optimizations; graph.Swap(graph_def); } diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h index 939e5fa046..a0748abfe6 100644 --- a/tensorflow/core/grappler/grappler_item.h +++ b/tensorflow/core/grappler/grappler_item.h @@ -77,6 +77,15 @@ struct GrapplerItem { // Return a set of node names that must be preserved. This includes feed and // fetch nodes, keep_ops, init_ops. std::unordered_set NodesToPreserve() const; + + // Restrict types of optimizations that are allowed for this GrapplerItem. + struct AllowedOptimizations { + // Is it allowed to add nodes to the graph that do not have registered + // gradient function. + bool non_differentiable_rewrites = true; + }; + + AllowedOptimizations allowed_optimizations; }; // Return the transitive fanin of a set of terminal nodes. diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index 3521669b63..9f0d9dbf28 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -425,6 +425,10 @@ bool IsSwitch(const NodeDef& node) { return op == "Switch" || op == "RefSwitch"; } +bool IsSymbolicGradient(const NodeDef& node) { + return node.op() == "SymbolicGradient"; +} + bool IsTanhGrad(const NodeDef& node) { return node.op() == "TanhGrad"; } bool IsTile(const NodeDef& node) { return node.op() == "Tile"; } diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index 25ab6b65ac..7f86a5f295 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -149,6 +149,7 @@ bool IsStridedSliceGrad(const NodeDef& node); bool IsSub(const NodeDef& node); bool IsSum(const NodeDef& node); bool IsSwitch(const NodeDef& node); +bool IsSymbolicGradient(const NodeDef& node); bool IsTanhGrad(const NodeDef& node); bool IsTile(const NodeDef& node); bool IsTranspose(const NodeDef& node); diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 960d1addb3..c708f84948 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -525,6 +525,7 @@ cc_library( "//tensorflow/core:core_cpu_base", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler/utils:colocation", @@ -541,6 +542,7 @@ tf_cuda_cc_test( ":custom_graph_optimizer_registry", ":meta_optimizer", "//tensorflow/cc:cc_ops", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core:tensorflow", "//tensorflow/core:test", diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 3388ee8035..7d5014ee0a 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -3249,6 +3249,10 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/, optimized_graph_ = &optimized_item.graph; node_map_.reset(new NodeMap(optimized_graph_)); + // Disable restricted graph rewrites. + options_.unary_ops_composition &= + item.allowed_optimizations.non_differentiable_rewrites; + if (options_.dedup_computations) { DedupComputations(); } diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 406c1b60ce..a5f851fb1a 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/grappler/utils/functions.h" #include "tensorflow/core/grappler/utils/topological_sort.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/util/ptr_util.h" namespace tensorflow { @@ -413,6 +414,15 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, FunctionLibraryDefinition flib(OpRegistry::Global(), optimized_graph->library()); + // Find functions for which we might need to compute a gradient at runtime. + gtl::FlatSet differentiable_functions; + for (const NodeDef& node : optimized_graph->node()) { + if (IsSymbolicGradient(node)) { + const auto* f_attr = gtl::FindOrNull(node.attr(), "f"); + if (f_attr) differentiable_functions.insert(f_attr->func().name()); + } + } + // Optimize each function only once. std::unordered_set optimized_funcs; bool optimize_function_library = true; @@ -428,6 +438,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, // Skip parametrized functions (function type or body is defined only at // function call time by caller node attributes). + // They should be specialized to their instantiation type parameters by + // the function optimizer, before we can optimize function body. if (IsParametrized(func)) continue; VLOG(3) << "Optimize function: function=" << func_name; @@ -442,6 +454,13 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem( func, flib, item.graph.versions().producer(), &func_item)); + // If we need to compute the gradient of optimized function at runtime, we + // can't perform non-differentiable rewrites. + if (differentiable_functions.find(func_name) != + differentiable_functions.end()) { + func_item.allowed_optimizations.non_differentiable_rewrites = false; + } + // Optimize function body graph. GraphDef optimized_func_graph; TF_RETURN_IF_ERROR( diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc index c477c4d4b1..3f3f43382f 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/grappler/utils.h" #include "tensorflow/core/grappler/utils/grappler_test.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/platform/test.h" namespace tensorflow { @@ -82,6 +83,48 @@ class TestOptimizerWithParams : public TestOptimizer { REGISTER_GRAPH_OPTIMIZER(TestOptimizerWithParams); +// Record various properties of the GrapplerItems passed for optimization. +class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer { + public: + static void SetAllowedOptimizations( + gtl::FlatMap* + allowed_optimizations) { + allowed_optimizations_ = allowed_optimizations; + } + static void ResetAllowedOptimizations() { allowed_optimizations_ = nullptr; } + + GrapplerItemPropertiesAccumulator() {} + string name() const override { + return "grappler_item_properties_accumulator"; + } + + Status Init( + const tensorflow::RewriterConfig_CustomGraphOptimizer* config) override { + return Status::OK(); + } + + Status Optimize(Cluster* cluster, const GrapplerItem& item, + GraphDef* optimized_graph) override { + *optimized_graph = item.graph; + if (allowed_optimizations_) { + allowed_optimizations_->insert({item.id, item.allowed_optimizations}); + } + return Status::OK(); + } + + void Feedback(Cluster* cluster, const GrapplerItem& item, + const GraphDef& optimized_graph, double result) override {} + + private: + static gtl::FlatMap* + allowed_optimizations_; +}; + +gtl::FlatMap* + GrapplerItemPropertiesAccumulator::allowed_optimizations_; + +REGISTER_GRAPH_OPTIMIZER(GrapplerItemPropertiesAccumulator); + class MetaOptimizerTest : public GrapplerTest {}; TEST_F(MetaOptimizerTest, RunsCustomOptimizer) { @@ -335,6 +378,89 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) { test::ExpectTensorEqual(tensors_expected[1], tensors[1]); } +TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) { + using test::function::NDef; + using FDH = FunctionDefHelper; + + // We will record what type of optimizations meta optimizer allows for each + // GrapplerItem (main graph and graphs for each function). + gtl::FlatMap + allowed_optimizations; + GrapplerItemPropertiesAccumulator::SetAllowedOptimizations( + &allowed_optimizations); + + // Just record properties of optimized Grappler items. + RewriterConfig rewriter_config; + rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO); + rewriter_config.add_optimizers("GrapplerItemPropertiesAccumulator"); + rewriter_config.set_min_graph_nodes(-1); + + MetaOptimizer optimizer(nullptr, rewriter_config); + + // Define simple function library with two identical mul functions. + FunctionDef mul_func_1 = FunctionDefHelper::Create( + "MyMul1", {"x:float", "y:float"}, {"z:float"}, {}, + {{{"mul"}, "Mul", {"x", "y"}, {}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "mul:z:0"}}); + + FunctionDef mul_func_2 = FunctionDefHelper::Create( + "MyMul2", {"x:float", "y:float"}, {"z:float"}, {}, + {{{"mul"}, "Mul", {"x", "y"}, {}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "mul:z:0"}}); + + // Tensorflow graph: + // + // x0 = tf.Placeholder(tf.float); + // x1 = tf.Placeholder(tf.float); + // dy = tf.Placeholder(tf.float); + // + // mul_1 = MyMul1(x0, x1); + // mul_2 = MyMul2(x0, x1); + // dx = SymbolicGradient({x0, x1, dy}, f=MyMul2) + GrapplerItem item; + item.id = "main"; + item.graph = test::function::GDef( + {NDef("x0", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), + NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), + NDef("dy", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), + // Calls into function library + NDef("mul_1", "MyMul1", {"x0", "x1"}, {}, kDevice), + NDef("mul_2", "MyMul2", {"x0", "x1"}, {}, kDevice), + // Symbolic gradient of a MyMul2 + NDef("dx", "SymbolicGradient", {"x0", "x1", "dy"}, + {{"f", FDH::FunctionRef("MyMul2", {})}, + {"Tin", DataTypeSlice{DT_FLOAT}}, + {"Tout", DataTypeSlice{DT_FLOAT, DT_FLOAT}}}, + kDevice)}, + // FunctionLib + {mul_func_1, mul_func_2}); + item.fetch = {"mul_1", "mul_2", "dx"}; + + GraphDef output; + TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output)); + + // Our custom optimizer must be called for the main graph and for the two + // functions. + ASSERT_EQ(allowed_optimizations.size(), 3); + + auto allowed_optimizations_main = + gtl::FindOrNull(allowed_optimizations, "main"); + ASSERT_NE(allowed_optimizations_main, nullptr); + EXPECT_TRUE(allowed_optimizations_main->non_differentiable_rewrites); + + auto allowed_optimizations_my_mul_1 = + gtl::FindOrNull(allowed_optimizations, "MyMul1"); + ASSERT_NE(allowed_optimizations_my_mul_1, nullptr); + EXPECT_TRUE(allowed_optimizations_my_mul_1->non_differentiable_rewrites); + + auto allowed_optimizations_my_mul_2 = + gtl::FindOrNull(allowed_optimizations, "MyMul2"); + ASSERT_NE(allowed_optimizations_my_mul_2, nullptr); + EXPECT_FALSE(allowed_optimizations_my_mul_2->non_differentiable_rewrites); +} + } // namespace } // namespace grappler } // namespace tensorflow -- GitLab From b73c5f80926de3b724a92a57cf0bc49aa7de37bd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 05:50:51 -0700 Subject: [PATCH 0144/1085] Automated rollback of commit 3f4423fad57694bc8d7adc427d65e5a18c8592b2 PiperOrigin-RevId: 215200418 --- .../contrib/tpu/ops/tpu_embedding_ops.cc | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc index 6b0730b40c..5c27d59f82 100644 --- a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc +++ b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc @@ -103,10 +103,19 @@ Status RegisterPerTableLoadOpsForAlgorithmBody( arg->set_type(DT_FLOAT); } } + { + auto* table_id_attr = op_def->add_attr(); + table_id_attr->set_name("table_id"); + table_id_attr->set_type("int"); + table_id_attr->set_has_minimum(true); + table_id_attr->set_minimum(-1); + table_id_attr->mutable_default_value()->set_i(-1); + } { auto* table_name_attr = op_def->add_attr(); table_name_attr->set_name("table_name"); table_name_attr->set_type("string"); + table_name_attr->mutable_default_value()->set_s(""); } { auto* num_shards_attr = op_def->add_attr(); @@ -138,9 +147,11 @@ parameters that are loaded from a checkpoint before a training loop is executed. %s table_name: Name of this table; must match a name in the - EmbeddingLayerConfiguration proto. + EmbeddingLayerConfiguration proto (overrides table_id). num_shards: Number of shards into which the embedding tables are divided. shard_id: Identifier of shard for this operation. +table_id: Index of this table in the EmbeddingLayerConfiguration proto + (deprecated). )doc", parameter_descriptions.c_str())); op_def->set_is_commutative(false); @@ -149,10 +160,14 @@ shard_id: Identifier of shard for this operation. auto shape_inference_function = [state_variable_specs, is_debug_op](shape_inference::InferenceContext* c) -> Status { + int table_id; + TF_RETURN_IF_ERROR(c->GetAttr("table_id", &table_id)); string table_name; TF_RETURN_IF_ERROR(c->GetAttr("table_name", &table_name)); - if (table_name.empty()) { - return errors::InvalidArgument("table_name attribute must be set"); + // Exactly one must be non-default. + if ((table_id >= 0) == (!table_name.empty())) { + return errors::InvalidArgument( + "exactly one of table_id or table_name must be non-default"); } int num_shards; TF_RETURN_IF_ERROR(c->GetAttr("num_shards", &num_shards)); @@ -225,10 +240,19 @@ Status RegisterPerTableRetrieveOpsForAlgorithmBody( arg->set_type(DT_FLOAT); } } + { + auto* table_id_attr = op_def->add_attr(); + table_id_attr->set_name("table_id"); + table_id_attr->set_type("int"); + table_id_attr->set_has_minimum(true); + table_id_attr->set_minimum(-1); + table_id_attr->mutable_default_value()->set_i(-1); + } { auto* table_name_attr = op_def->add_attr(); table_name_attr->set_name("table_name"); table_name_attr->set_type("string"); + table_name_attr->mutable_default_value()->set_s(""); } { auto* num_shards_attr = op_def->add_attr(); @@ -259,9 +283,11 @@ the correct embedding table configuration. For example, this op is used to retrieve updated parameters before saving a checkpoint. %s table_name: Name of this table; must match a name in the - EmbeddingLayerConfiguration proto. + EmbeddingLayerConfiguration proto (overrides table_id). num_shards: Number of shards into which the embedding tables are divided. shard_id: Identifier of shard for this operation. +table_id: Index of this table in the EmbeddingLayerConfiguration proto + (deprecated). )doc", parameter_descriptions.c_str())); op_def->set_is_commutative(false); @@ -270,10 +296,14 @@ shard_id: Identifier of shard for this operation. auto shape_inference_function = [state_variable_specs, is_debug_op](shape_inference::InferenceContext* c) -> Status { + int table_id; + TF_RETURN_IF_ERROR(c->GetAttr("table_id", &table_id)); string table_name; TF_RETURN_IF_ERROR(c->GetAttr("table_name", &table_name)); - if (table_name.empty()) { - return errors::InvalidArgument("table_name must be non-empty"); + // Exactly one must be non-default. + if ((table_id >= 0) == (!table_name.empty())) { + return errors::InvalidArgument( + "exactly one of table_id or table_name must be non-default"); } int num_shards; TF_RETURN_IF_ERROR(c->GetAttr("num_shards", &num_shards)); -- GitLab From 7c5eb354a6b5b2d5a2e27d8ce3dc4861cb51153c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 07:15:23 -0700 Subject: [PATCH 0145/1085] In TensorFlow configure, write the .tf_configure.bazelrc into the --workspace path if provided. This allows repositories that depend on TensorFlow to execute 'bazel run @org_tensorflow//:configure -- --workspace $(pwd)' to configure TensorFlow. END_PUBLIC Before this change, the .tf_configure.bazelrc ended up in the bazel exec root, and 'bazel clean' would undo the configuration. PiperOrigin-RevId: 215209207 --- configure.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/configure.py b/configure.py index 0a3b9a7894..796c6231e8 100644 --- a/configure.py +++ b/configure.py @@ -48,10 +48,9 @@ _SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16] _DEFAULT_PROMPT_ASK_ATTEMPTS = 10 -_TF_WORKSPACE_ROOT = os.path.abspath(os.path.dirname(__file__)) _TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' -_TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) -_TF_WORKSPACE = os.path.join(_TF_WORKSPACE_ROOT, 'WORKSPACE') +_TF_WORKSPACE_ROOT = '' +_TF_BAZELRC = '' if platform.machine() == 'ppc64le': _DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/powerpc64le-linux-gnu/' @@ -243,10 +242,10 @@ def setup_python(environ_cp): f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) -def reset_tf_configure_bazelrc(workspace_path): +def reset_tf_configure_bazelrc(): """Reset file that contains customized config settings.""" open(_TF_BAZELRC, 'w').close() - bazelrc_path = os.path.join(workspace_path, '.bazelrc') + bazelrc_path = os.path.join(_TF_WORKSPACE_ROOT, '.bazelrc') data = [] if os.path.exists(bazelrc_path): @@ -1469,21 +1468,27 @@ def config_info_line(name, help_text): def main(): + global _TF_WORKSPACE_ROOT + global _TF_BAZELRC + parser = argparse.ArgumentParser() parser.add_argument( '--workspace', type=str, - default=_TF_WORKSPACE_ROOT, + default=os.path.abspath(os.path.dirname(__file__)), help='The absolute path to your active Bazel workspace.') args = parser.parse_args() + _TF_WORKSPACE_ROOT = args.workspace + _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) + # Make a copy of os.environ to be clear when functions and getting and setting # environment variables. environ_cp = dict(os.environ) check_bazel_version('0.15.0') - reset_tf_configure_bazelrc(args.workspace) + reset_tf_configure_bazelrc() cleanup_makefile() setup_python(environ_cp) -- GitLab From 9a2f872acd0c38d74d60e4f67701241aa1a26419 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 08:21:58 -0700 Subject: [PATCH 0146/1085] Move from deprecated self.test_session() to self.cached_session() or self.session(). * Move from self.test_session(graph=ops.Graph(), ...) to self.session(...) (semantically equivalent). * Move from self.test_session() to self.cached_session(config=self.config) when run_in_graph_and_eager_modes(config=config) is set to be consistent between eager and non eager modes. self.test_session() has been deprecated in 9962eb5e84b15e309410071b06c2ed2d6148ed44 as its name confuses readers of the test. Moving to cached_session() instead which is more explicit about: * the fact that the session may be reused. * the session is not closed even when doing a "with self.test_session()" statement. PiperOrigin-RevId: 215216964 --- tensorflow/contrib/distribute/python/values_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py index ae3e134333..121d2fbb3f 100644 --- a/tensorflow/contrib/distribute/python/values_test.py +++ b/tensorflow/contrib/distribute/python/values_test.py @@ -641,7 +641,7 @@ class MirroredVariableTest(test.TestCase): if context.num_gpus() < 1 and context.executing_eagerly(): self.skipTest("A GPU is not available for this test in eager mode.") - with self.test_session() as sess: + with self.cached_session(config=self.config) as sess: v, devices, mirrored = _make_mirrored() # Overwrite the initial values. @@ -744,7 +744,7 @@ class MirroredVariableTest(test.TestCase): if context.num_gpus() < 1 or context.executing_eagerly(): self.skipTest("A GPU is not available for this test or it's eager mode.") - with self.test_session( + with self.session( graph=ops.Graph()) as sess, mirrored_strategy.MirroredStrategy( ["/device:GPU:0"]).scope(): with ops.device("/device:GPU:0"): @@ -827,7 +827,7 @@ class TowerLocalVariableTest(test.TestCase): if context.num_gpus() < 1 and context.executing_eagerly(): self.skipTest("A GPU is not available for this test in eager mode.") - with self.test_session() as sess: + with self.cached_session(config=self.config) as sess: v, tower_local = _make_tower_local(variable_scope.VariableAggregation.SUM) # Overwrite the initial values. @@ -850,7 +850,7 @@ class TowerLocalVariableTest(test.TestCase): if context.num_gpus() < 1 and context.executing_eagerly(): self.skipTest("A GPU is not available for this test in eager mode.") - with self.test_session() as sess: + with self.cached_session(config=self.config) as sess: v, tower_local = _make_tower_local( variable_scope.VariableAggregation.MEAN) -- GitLab From e285dea8d9626b832f34d65159639f294c2d6881 Mon Sep 17 00:00:00 2001 From: Shashi Shekhar Date: Mon, 1 Oct 2018 09:23:48 -0700 Subject: [PATCH 0147/1085] Update documentation. - Use absolute links instead of relative links. Relative links break when published on website. - Correct NNAPI abbreviation. PiperOrigin-RevId: 215225415 --- tensorflow/contrib/lite/g3doc/performance.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/lite/g3doc/performance.md b/tensorflow/contrib/lite/g3doc/performance.md index 0ae9400068..6b7943caf8 100644 --- a/tensorflow/contrib/lite/g3doc/performance.md +++ b/tensorflow/contrib/lite/g3doc/performance.md @@ -7,12 +7,12 @@ Mobile and embedded devices have limited computational resources and it is impor Some models may be too large to run on embedded devices. Instead of large models it is better to use a slightly less precise but smaller model for embedded devices. Smaller models not only use less disk space and memory but are generally faster and more energy efficient. One example of models optimized for mobile devices are [MobileNets](https://arxiv.org/abs/1704.04861), which are optimized for mobile vision applications. Tensorflow Lite [models page](models.md) lists several other models that have been optimized specifically for mobile and embedded devices. You can retrain the listed models on your own dataset by using transfer learning. Check out our transfer learning tutorial for -[image classification] (https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) and +[image classification](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) and [object detection](https://medium.com/tensorflow/training-and-serving-a-realtime-mobile-object-detector-in-30-minutes-with-cloud-tpus-b78971cf1193). ## Profile your model -Before starting any optimization, it is a good practice to profile and benchmark your model. Tensorflow Lite [benchmarking tool](../tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time. +Before starting any optimization, it is a good practice to profile and benchmark your model. Tensorflow Lite [benchmarking tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time. ## Profile and optimize operators in the graph If a particular operator appears frequently in the model and based on profiling you find the operator consuming the most amount of time, you can look into optimizing the operator. @@ -22,7 +22,7 @@ If a particular operator appears frequently in the model and based on profiling If your model uses floating point weights or activations then it may be possible to reduce the size of model up to ~4x by using quantization and other model optimizations. Check out our [model optimization toolkit](https://www.tensorflow.org/performance/model_optimization) for details about optimizing your model. Fully quantized models can be remarkably power efficient as well. ## Tweak the number of threads -Tensorflow Lite supports multi-threaded kernels for many operators. You can increase the number of threads and speed up execution of operators. Increasing the number of threads will however make your model use more resources and power. For some applications latency may be more important than energy efficiency. You can increase the number of threads by setting the number of [interpreter](../interpreter.h) threads. +Tensorflow Lite supports multi-threaded kernels for many operators. You can increase the number of threads and speed up execution of operators. Increasing the number of threads will however make your model use more resources and power. For some applications latency may be more important than energy efficiency. You can increase the number of threads by setting the number of [interpreter](https://github.com/tensorflow/tensorflow/blob/1084594657a5d139102ac794f84d1427a710e39a/tensorflow/contrib/lite/interpreter.h#L337) threads. ## Eliminate redundant copies Tensorflow Lite is optimized to reduce redundant copies. The APIs allow user to [mmap a model file](https://github.com/tensorflow/tensorflow/blob/9982fd6c8831cbd2f58954f79ea71f26660393bc/tensorflow/contrib/lite/model.h#L152) and avoid copies. If your application is not careful, there can be redundant copies when feeding the input to the model and reading output from the model. Make sure to eliminate redundant copies. If you are using higher level APIs like Java API, make sure to carefully check the documentation for performance caveats. For example, the Java API is a lot faster if ByteBuffers are used as [inputs](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L151). @@ -31,8 +31,8 @@ Tensorflow Lite is optimized to reduce redundant copies. The APIs allow user to Platform specific tools like [Android profiler](https://developer.android.com/studio/profile/android-profiler) and [Instruments](https://help.apple.com/instruments/mac/current/) provide a wealth of profiling information that can be used to debug your app. Sometimes the performance bug may be not in the model but in parts of application code that interact with the model. Make sure to familiarize yourself with platform specific profiling tools and best practices for your platform. ## Use hardware accelerators available on the device -Tensorflow Lite is working on adding support for accelerators like GPU and provides acceleration through [NNAPI](https://developer.android.com/ndk/guides/neuralnetworks/) on Android. -You can utilize these hardware accelerator backends to improve the speed and efficiency of your model. To enable NNAPI call [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/contrib/lite/interpreter.h#L334) on the interpreter instance. +Tensorflow Lite is working on adding support for accelerators like GPU and provides acceleration through [Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/) on Android. +You can utilize these hardware accelerator backends to improve the speed and efficiency of your model. To enable Neural Networks API call [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/contrib/lite/interpreter.h#L334) on the interpreter instance. ## Need more help The Tensorflow team is happy to help diagnose and address specific performance issues you may be facing. Please file a bug on [github](https://github.com/tensorflow/tensorflow/issues) with details of the issue. -- GitLab From 03a18ca576410d49e8f0692464e35e900a54f59f Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Mon, 1 Oct 2018 10:01:20 -0700 Subject: [PATCH 0148/1085] Remove outdated integration test in preparation for update of keras_preprocessing. PiperOrigin-RevId: 215231309 --- .../python/keras/preprocessing/image_test.py | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/tensorflow/python/keras/preprocessing/image_test.py b/tensorflow/python/keras/preprocessing/image_test.py index 362cbc1dc9..4abaadfcd3 100644 --- a/tensorflow/python/keras/preprocessing/image_test.py +++ b/tensorflow/python/keras/preprocessing/image_test.py @@ -94,43 +94,6 @@ class TestImage(test.TestCase): self.assertEqual(x.shape[1:], images.shape[1:]) break - def test_image_data_generator_with_validation_split(self): - if PIL is None: - return # Skip test if PIL is not available. - - for test_images in _generate_test_images(): - img_list = [] - for im in test_images: - img_list.append(keras.preprocessing.image.img_to_array(im)[None, ...]) - - images = np.vstack(img_list) - generator = keras.preprocessing.image.ImageDataGenerator( - validation_split=0.5) - seq = generator.flow( - images, - np.arange(images.shape[0]), - shuffle=False, - batch_size=3, - subset='validation') - _, y = seq[0] - self.assertEqual(list(y), [0, 1, 2]) - seq = generator.flow( - images, - np.arange(images.shape[0]), - shuffle=False, - batch_size=3, - subset='training') - _, y2 = seq[0] - self.assertEqual(list(y2), [4, 5, 6]) - - with self.assertRaises(ValueError): - generator.flow( - images, - np.arange(images.shape[0]), - shuffle=False, - batch_size=3, - subset='foo') - def test_image_data_generator_with_split_value_error(self): with self.assertRaises(ValueError): keras.preprocessing.image.ImageDataGenerator(validation_split=5) -- GitLab From a5fc8b064884b926ade9f7973dc096c0677a14e0 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Mon, 1 Oct 2018 10:35:02 -0700 Subject: [PATCH 0149/1085] Name fusion parameters simply "param_X". Where "X" is the parameter number. Previously, fusion parameter names including the name of the original instruction which produced the value which was confusing. PiperOrigin-RevId: 215238171 --- .../compiler/xla/service/hlo_computation.cc | 36 +++---------------- .../compiler/xla/service/hlo_instructions.cc | 3 +- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 0e5920af7a..4613d6762e 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -122,30 +122,6 @@ HloInstruction* HloComputation::AddParameter( return instructions_.back().get(); } -namespace { - -// Returns the new name for a fusion parameter when we change its number. -// -// Fusion parameters are named foo.param_1, bar.param_2, etc. We are -// renumbering the parameters, so replace the final number in the name with -// the updated value. -string RenameFusionParameter(const string& original_name, int64 new_param_no) { - const string param_underscore = ".param_"; - size_t index = original_name.rfind(param_underscore); - if (index == string::npos) { - return original_name; - } - string after_param = original_name.substr(index + param_underscore.size()); - int64 numeric_suffix; - if (absl::SimpleAtoi(after_param, &numeric_suffix)) { - return StrCat(original_name.substr(0, index + param_underscore.size()), - new_param_no); - } - return original_name; -} - -} // namespace - Status HloComputation::RemoveParameter(int64 param_no) { CHECK_GE(param_no, 0); CHECK_LT(param_no, param_instructions_.size()); @@ -158,11 +134,9 @@ Status HloComputation::RemoveParameter(int64 param_no) { while (param_no < param_instructions_.size()) { param_instruction = param_instructions_[param_no]; - string param_name = - RenameFusionParameter(param_instruction->name(), param_no); HloInstruction* new_instr = AddInstructionInternal(HloInstruction::CreateParameter( - param_no, param_instruction->shape(), param_name)); + param_no, param_instruction->shape(), StrCat("param_", param_no))); TF_RETURN_IF_ERROR(param_instruction->ReplaceAllUsesWith(new_instr)); param_instructions_[param_no] = new_instr; TF_RETURN_IF_ERROR(RemoveInstruction(param_instruction)); @@ -186,11 +160,9 @@ Status HloComputation::RemoveUnusedParameters() { if (removed > 0) { const int64 param_no = i - removed; - string param_name = - RenameFusionParameter(param_instruction->name(), param_no); - HloInstruction* new_instr = - AddInstructionInternal(HloInstruction::CreateParameter( - param_no, param_instruction->shape(), param_name)); + HloInstruction* new_instr = AddInstructionInternal( + HloInstruction::CreateParameter(param_no, param_instruction->shape(), + StrCat("param_", param_no))); TF_RETURN_IF_ERROR(param_instruction->ReplaceAllUsesWith(new_instr)); param_instructions_[param_no] = new_instr; TF_RETURN_IF_ERROR(RemoveInstruction(param_instruction)); diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index cd71bc3323..ad45a82941 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -1042,7 +1042,8 @@ HloInstruction* HloFusionInstruction::AddFusionOperand( const int64 param_no = operand_count(); // Name the parameter after the instruction it represents in the outer // (non-fusion) computation. - string param_name = StrCat(new_operand->name(), ".param_", param_no); + // string param_name = StrCat(new_operand->name(), ".param_", param_no); + string param_name = StrCat("param_", param_no); HloInstruction* fused_parameter = fused_instructions_computation()->AddParameter( HloInstruction::CreateParameter(param_no, new_operand->shape(), -- GitLab From a6478312ef296ba9684931135851e9c7bb460444 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 1 Oct 2018 10:36:07 -0700 Subject: [PATCH 0150/1085] Replace the tf.name_scope call with an internal context manager that can contain additional boilerplate later on. Unfortunately it could not be extended to include the error handling. PiperOrigin-RevId: 215238369 --- tensorflow/python/autograph/converters/BUILD | 6 +-- .../{name_scopes.py => function_scopes.py} | 32 ++++++++------- ...scopes_test.py => function_scopes_test.py} | 40 +++++++++---------- tensorflow/python/autograph/core/BUILD | 12 ++++++ .../autograph/core/converter_testing.py | 2 + .../autograph/core/function_wrapping.py | 30 ++++++++++++++ .../autograph/core/function_wrapping_test.py | 34 ++++++++++++++++ .../python/autograph/impl/conversion.py | 6 ++- 8 files changed, 122 insertions(+), 40 deletions(-) rename tensorflow/python/autograph/converters/{name_scopes.py => function_scopes.py} (72%) rename tensorflow/python/autograph/converters/{name_scopes_test.py => function_scopes_test.py} (71%) create mode 100644 tensorflow/python/autograph/core/function_wrapping.py create mode 100644 tensorflow/python/autograph/core/function_wrapping_test.py diff --git a/tensorflow/python/autograph/converters/BUILD b/tensorflow/python/autograph/converters/BUILD index 7b029de8ed..f06dc78f0e 100644 --- a/tensorflow/python/autograph/converters/BUILD +++ b/tensorflow/python/autograph/converters/BUILD @@ -27,10 +27,10 @@ py_library( "decorators.py", "directives.py", "error_handlers.py", + "function_scopes.py", "list_comprehensions.py", "lists.py", "logical_expressions.py", - "name_scopes.py", "return_statements.py", "side_effect_guards.py", "slices.py", @@ -157,8 +157,8 @@ py_test( ) py_test( - name = "name_scopes_test", - srcs = ["name_scopes_test.py"], + name = "function_scopes_test", + srcs = ["function_scopes_test.py"], deps = [ ":converters", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/autograph/converters/name_scopes.py b/tensorflow/python/autograph/converters/function_scopes.py similarity index 72% rename from tensorflow/python/autograph/converters/name_scopes.py rename to tensorflow/python/autograph/converters/function_scopes.py index a9c55ccff0..284b5b3519 100644 --- a/tensorflow/python/autograph/converters/name_scopes.py +++ b/tensorflow/python/autograph/converters/function_scopes.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Wraps a function body with a `name_scope` of the function name.""" +"""Wraps the body of a converted function with auxiliary constructs.""" from __future__ import absolute_import from __future__ import division @@ -24,8 +24,8 @@ from tensorflow.python.autograph.core import converter from tensorflow.python.autograph.pyct import templates -class FunctionNameScopeTransformer(converter.Base): - """Wrap a function body with a `name_scope` of the function name.""" +class FunctionBodyTransformer(converter.Base): + """Wraps function bodies around autograph-specific boilerplate.""" def _name_for_current_scope(self): innermost = self.enclosing_entities[-1] @@ -49,26 +49,28 @@ class FunctionNameScopeTransformer(converter.Base): def visit_FunctionDef(self, node): node = self.generic_visit(node) - unscoped_body = [] - scoped_body = node.body - if scoped_body: - first = scoped_body[0] - if isinstance(first, gast.Expr) and isinstance(first.value, gast.Str): - # Skip any docstring. - unscoped_body = scoped_body[:1] - scoped_body = scoped_body[1:] + final_body = [] + indented_body = node.body + if node.body: + first_statement = node.body[0] + # Skip the docstring, if any. + if (isinstance(first_statement, gast.Expr) and + isinstance(first_statement.value, gast.Str)): + indented_body = indented_body[1:] + final_body.append(first_statement) template = """ - with tf.name_scope(scope_name): + with ag__.function_scope(scope_name): body """ scoped_body = templates.replace( template, scope_name=gast.Str(self._name_for_current_scope()), - body=scoped_body) - node.body = unscoped_body + scoped_body + body=indented_body) + final_body.extend(scoped_body) + node.body = final_body return node def transform(node, ctx): - return FunctionNameScopeTransformer(ctx).visit(node) + return FunctionBodyTransformer(ctx).visit(node) diff --git a/tensorflow/python/autograph/converters/name_scopes_test.py b/tensorflow/python/autograph/converters/function_scopes_test.py similarity index 71% rename from tensorflow/python/autograph/converters/name_scopes_test.py rename to tensorflow/python/autograph/converters/function_scopes_test.py index 73933c1c4f..e5ce03a109 100644 --- a/tensorflow/python/autograph/converters/name_scopes_test.py +++ b/tensorflow/python/autograph/converters/function_scopes_test.py @@ -12,51 +12,51 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for for_canonicalization module.""" +"""Tests for function_scopes module.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.autograph.converters import name_scopes +from tensorflow.python.autograph.converters import function_scopes from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.platform import test -class FunctionNameScopeTransformer(converter_testing.TestCase): +class FunctionBodyTransformerTest(converter_testing.TestCase): def test_basic(self): def test_fn(l): - """This should stay here.""" + """Docstring.""" a = 1 l += a return l - with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: + with self.converted(test_fn, function_scopes, {}) as result: result_op = result.test_fn(constant_op.constant(1)) self.assertIn('test_fn/', result_op.op.name) - self.assertEqual('This should stay here.', result.test_fn.__doc__) + self.assertEqual('Docstring.', result.test_fn.__doc__) - def test_long_docstring(self): + def test_multiline_docstring(self): - def test_fn(l): - """Multi-line docstring. + tf = None + + def test_fn(): + """First sentence. - Args: - l: A thing. - Returns: - l + Second sentence. """ - return l + 1 + return tf.constant(1) - with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: - result_op = result.test_fn(constant_op.constant(1)) + with self.converted(test_fn, function_scopes, {}, + constant_op.constant) as result: + result_op = result.test_fn() self.assertIn('test_fn/', result_op.op.name) - self.assertIn('Multi-line docstring.', result.test_fn.__doc__) - self.assertIn('Returns:', result.test_fn.__doc__) + self.assertIn('First sentence.', result.test_fn.__doc__) + self.assertIn('Second sentence.', result.test_fn.__doc__) def test_nested_functions(self): @@ -68,7 +68,7 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): l += 1 return l, inner_fn(l) - with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: + with self.converted(test_fn, function_scopes, {}, ops.name_scope) as result: first, second = result.test_fn(constant_op.constant(1)) self.assertIn('test_fn/', first.op.name) self.assertNotIn('inner_fn', first.op.name) @@ -88,7 +88,7 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): ns = {'TestClass': TestClass} node, ctx = self.prepare(TestClass, ns, owner_type=TestClass) - node = name_scopes.transform(node, ctx) + node = function_scopes.transform(node, ctx) with self.compiled(node, {}, ops.name_scope) as result: first, second = result.TestClass().test_fn(constant_op.constant(1)) diff --git a/tensorflow/python/autograph/core/BUILD b/tensorflow/python/autograph/core/BUILD index 85fecf084d..843e381f31 100644 --- a/tensorflow/python/autograph/core/BUILD +++ b/tensorflow/python/autograph/core/BUILD @@ -20,11 +20,13 @@ py_library( "config.py", "converter.py", "errors.py", + "function_wrapping.py", "naming.py", ], srcs_version = "PY2AND3", visibility = ["//tensorflow:__subpackages__"], deps = [ + "//tensorflow/python:framework_ops", "//tensorflow/python/autograph/pyct", "//tensorflow/python/autograph/pyct/static_analysis", "//tensorflow/python/autograph/utils", @@ -46,6 +48,16 @@ py_test( ], ) +py_test( + name = "function_wrapping_test", + srcs = ["function_wrapping_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":core", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "naming_test", srcs = ["naming_test.py"], diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py index 7ce1b7c4c5..dc2d419d34 100644 --- a/tensorflow/python/autograph/core/converter_testing.py +++ b/tensorflow/python/autograph/core/converter_testing.py @@ -29,6 +29,7 @@ from tensorflow.python.autograph import utils from tensorflow.python.autograph.core import config from tensorflow.python.autograph.core import converter from tensorflow.python.autograph.core import errors +from tensorflow.python.autograph.core import function_wrapping from tensorflow.python.autograph.pyct import compiler from tensorflow.python.autograph.pyct import parser from tensorflow.python.autograph.pyct import pretty_printer @@ -112,6 +113,7 @@ class TestCase(test.TestCase): fake_ag.__dict__['utils'] = utils fake_ag.__dict__['rewrite_graph_construction_error'] = ( errors.rewrite_graph_construction_error) + fake_ag.__dict__['function_scope'] = function_wrapping.function_scope result.__dict__['ag__'] = fake_ag for k, v in namespace.items(): result.__dict__[k] = v diff --git a/tensorflow/python/autograph/core/function_wrapping.py b/tensorflow/python/autograph/core/function_wrapping.py new file mode 100644 index 0000000000..21b66eff02 --- /dev/null +++ b/tensorflow/python/autograph/core/function_wrapping.py @@ -0,0 +1,30 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Support for wrapping converted functions bodies with auxiliary logic.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib + +from tensorflow.python.framework import ops + + +@contextlib.contextmanager +def function_scope(function_name): + """Returns a context manager for the converted body of a function.""" + with ops.name_scope(function_name): + yield diff --git a/tensorflow/python/autograph/core/function_wrapping_test.py b/tensorflow/python/autograph/core/function_wrapping_test.py new file mode 100644 index 0000000000..5e217055c7 --- /dev/null +++ b/tensorflow/python/autograph/core/function_wrapping_test.py @@ -0,0 +1,34 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for function_wrapping module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.autograph.core import function_wrapping +from tensorflow.python.framework import constant_op +from tensorflow.python.platform import test + + +class FunctionWrappingTest(test.TestCase): + + def test_function_scope_name(self): + with function_wrapping.function_scope('test_name'): + t = constant_op.constant(1) + self.assertIn('test_name', t.name) + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py index a0d13c82a8..52abd40626 100644 --- a/tensorflow/python/autograph/impl/conversion.py +++ b/tensorflow/python/autograph/impl/conversion.py @@ -34,15 +34,16 @@ from tensorflow.python.autograph.converters import control_flow from tensorflow.python.autograph.converters import decorators from tensorflow.python.autograph.converters import directives from tensorflow.python.autograph.converters import error_handlers +from tensorflow.python.autograph.converters import function_scopes from tensorflow.python.autograph.converters import lists from tensorflow.python.autograph.converters import logical_expressions -from tensorflow.python.autograph.converters import name_scopes from tensorflow.python.autograph.converters import return_statements from tensorflow.python.autograph.converters import side_effect_guards from tensorflow.python.autograph.converters import slices from tensorflow.python.autograph.core import config from tensorflow.python.autograph.core import converter from tensorflow.python.autograph.core import errors +from tensorflow.python.autograph.core import function_wrapping from tensorflow.python.autograph.pyct import ast_util from tensorflow.python.autograph.pyct import inspect_utils from tensorflow.python.autograph.pyct import origin_info @@ -257,6 +258,7 @@ def _add_self_references(namespace, autograph_module): ag_internal.converted_call = autograph_module.converted_call ag_internal.ConversionOptions = autograph_module.ConversionOptions ag_internal.utils = utils + ag_internal.function_scope = function_wrapping.function_scope ag_internal.rewrite_graph_construction_error = ( errors.rewrite_graph_construction_error) # TODO(mdan): Add safeguards against name clashes. @@ -346,7 +348,7 @@ def node_to_graph(node, context, rewrite_errors=True): node = converter.apply_(node, context, conditional_expressions) node = converter.apply_(node, context, logical_expressions) node = converter.apply_(node, context, side_effect_guards) - node = converter.apply_(node, context, name_scopes) + node = converter.apply_(node, context, function_scopes) if rewrite_errors: node = converter.apply_(node, context, error_handlers) return node -- GitLab From 57a831d20929e71279d164905fed93e1f518ee37 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 10:41:58 -0700 Subject: [PATCH 0151/1085] Bugfix: When a subgraph is encapsulated and replaced by XlaLaunch op, the requested device placement of the XlaLaunch op must be derived from the subgraph. PiperOrigin-RevId: 215239672 --- tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc | 6 ++++++ .../compiler/jit/encapsulate_xla_computations_pass.cc | 2 ++ .../jit/encapsulate_xla_computations_pass_test.cc | 9 ++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc index e0632ff7e4..15faf31077 100644 --- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc @@ -748,6 +748,12 @@ Node* Encapsulator::Subgraph::MakeNodeImage(const Graph* graph_in, Node* node) { graph_->set_versions(graph_in->versions()); } + // TODO(b/116981129): Enhance how the device for the encapsulated subgraph is + // determined. In case of hard placement, ensure all the encapsulated nodes + // have the same requested device, which in turn will be the requested device + // for the entire encapsulated subgraph. In case of soft placement, use a + // deterministic approach to fill in the requested device. Handle co-location + // constraints similarly if they exist. if (device_.empty()) { device_ = node->assigned_device_name().empty() ? node->requested_device() diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc index 97ef8cd3cb..755c364c62 100644 --- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc @@ -297,7 +297,9 @@ Status RewriteSubgraph(const std::vector& arg_source_tensors, // Target the XLA CPU/GPU backends. VLOG(2) << "Replacing with XlaLaunch"; + VLOG(2) << "Device is " << launch->requested_device(); def.set_op("XlaLaunch"); + def.set_device(launch->requested_device()); AddNodeAttr("Tconstants", DataTypeVector{}, &def); AddNodeAttr("Targs", arg_types, &def); AddNodeAttr("Nresources", num_variables, &def); diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass_test.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass_test.cc index f643fb0cfe..479038ac8e 100644 --- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass_test.cc +++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass_test.cc @@ -55,6 +55,7 @@ static std::unique_ptr MakeOuterGraph( .Input(u.node()->name(), 0, DT_RESOURCE) .Input(v.node()->name(), 0, DT_RESOURCE) .Input(w.node()->name(), 0, DT_RESOURCE) + .Device("/gpu:0") .Attr(EncapsulateXlaComputationsPass::kXlaClusterAttr, "launch0") .Attr("_variable_start_index", 4) .Finalize(&def)); @@ -107,10 +108,11 @@ static std::unique_ptr MakeBodyGraph() { auto add_attrs = [](Node* node) { node->AddAttr(EncapsulateXlaComputationsPass::kXlaClusterAttr, "launch0"); + node->set_requested_device("/gpu:0"); }; auto b_identity = ops::Identity(scope.WithOpName("B_identity"), arg1); - + add_attrs(b_identity.node()); auto read_u = ops::ReadVariableOp(scope.WithOpName("ReadU"), arg4, DT_FLOAT); add_attrs(read_u.node()); auto read_v = ops::ReadVariableOp(scope.WithOpName("ReadV"), arg5, DT_FLOAT); @@ -215,6 +217,7 @@ TEST(EncapsulateXlaComputations, Encapsulate) { auto add_attrs = [](Node* node) { node->AddAttr(EncapsulateXlaComputationsPass::kXlaClusterAttr, "launch0"); + node->set_requested_device("/gpu:0"); }; auto b_identity = ops::Identity(scope.WithOpName("B_identity"), b); @@ -317,8 +320,8 @@ TEST(EncapsulateXlaComputations, BuildXlaLaunchOp) { NameAttrList function; function.set_name("launch0"); auto launch = ops::XlaLaunch( - scope.WithOpName("launch0"), std::initializer_list{}, - std::initializer_list{a, b, c, d}, + scope.WithOpName("launch0").WithDevice("/gpu:0"), + std::initializer_list{}, std::initializer_list{a, b, c, d}, std::initializer_list{u, v, w}, DataTypeVector{DT_FLOAT, DT_INT32, DT_FLOAT, DT_FLOAT}, function); -- GitLab From ec2b5f889fb3eb677f7b8198cbd8d505b2779fa7 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 1 Oct 2018 10:42:14 -0700 Subject: [PATCH 0152/1085] Automated rollback of commit 5f822d694af6e4aa57fe8a426032a91dc61e30d6 PiperOrigin-RevId: 215239710 --- tensorflow/contrib/factorization/BUILD | 9 +-------- .../contrib/factorization/python/ops/gmm_ops.py | 14 +++++++------- .../factorization/python/ops/wals_test.py | 16 ++++++++-------- tensorflow/contrib/opt/BUILD | 5 ----- .../contrib/timeseries/python/timeseries/BUILD | 7 +------ 5 files changed, 17 insertions(+), 34 deletions(-) diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD index 510f292508..e344d7a23b 100644 --- a/tensorflow/contrib/factorization/BUILD +++ b/tensorflow/contrib/factorization/BUILD @@ -154,8 +154,6 @@ tf_py_test( ], tags = [ "no_pip", # b/38283730 - "noasan", # b/116875897 - "nomsan", "notsan", # Flaky: b/30756419 ], ) @@ -179,11 +177,7 @@ tf_py_test( "//tensorflow/python:random_seed", "//tensorflow/python:variables", ], - tags = [ - "noasan", # b/116875897 - "nomsan", - "notsan", # b/62863147 - ], + tags = ["notsan"], # b/62863147 ) py_library( @@ -282,7 +276,6 @@ tf_py_test( "manual", "noasan", # times out b/63678675 "nomsan", - "notsan", # b/116875897 ], ) diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py index e076631bc1..d365ad1117 100644 --- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py +++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py @@ -154,10 +154,10 @@ class GmmAlgorithm(object): def _create_variables(self): """Initializes GMM algorithm.""" init_value = array_ops.constant([], dtype=dtypes.float32) - self._means = variables.Variable(init_value, - name=self.CLUSTERS_VARIABLE, - validate_shape=False) - self._covs = variables.Variable( + self._means = variables.VariableV1(init_value, + name=self.CLUSTERS_VARIABLE, + validate_shape=False) + self._covs = variables.VariableV1( init_value, name=self.CLUSTERS_COVS_VARIABLE, validate_shape=False) # Mixture weights, representing the probability that a randomly # selected unobservable data (in EM terms) was generated by component k. @@ -165,9 +165,9 @@ class GmmAlgorithm(object): array_ops.tile([1.0 / self._num_classes], [self._num_classes]), name=self.CLUSTERS_WEIGHT, validate_shape=False) - self._cluster_centers_initialized = variables.Variable(False, - dtype=dtypes.bool, - name='initialized') + self._cluster_centers_initialized = variables.VariableV1(False, + dtype=dtypes.bool, + name='initialized') def _initialize_variables(self, data, initial_means=None): """Initializes variables. diff --git a/tensorflow/contrib/factorization/python/ops/wals_test.py b/tensorflow/contrib/factorization/python/ops/wals_test.py index 9bdbd05015..75d577f429 100644 --- a/tensorflow/contrib/factorization/python/ops/wals_test.py +++ b/tensorflow/contrib/factorization/python/ops/wals_test.py @@ -420,13 +420,13 @@ class WALSMatrixFactorizationUnsupportedTest(test.TestCase): class SweepHookTest(test.TestCase): def test_sweeps(self): - is_row_sweep_var = variables.Variable(True) - is_sweep_done_var = variables.Variable(False) - init_done = variables.Variable(False) - row_prep_done = variables.Variable(False) - col_prep_done = variables.Variable(False) - row_train_done = variables.Variable(False) - col_train_done = variables.Variable(False) + is_row_sweep_var = variables.VariableV1(True) + is_sweep_done_var = variables.VariableV1(False) + init_done = variables.VariableV1(False) + row_prep_done = variables.VariableV1(False) + col_prep_done = variables.VariableV1(False) + row_train_done = variables.VariableV1(False) + col_train_done = variables.VariableV1(False) init_op = state_ops.assign(init_done, True) row_prep_op = state_ops.assign(row_prep_done, True) @@ -486,7 +486,7 @@ class StopAtSweepHookTest(test.TestCase): def test_stop(self): hook = wals_lib._StopAtSweepHook(last_sweep=10) - completed_sweeps = variables.Variable( + completed_sweeps = variables.VariableV1( 8, name=wals_lib.WALSMatrixFactorization.COMPLETED_SWEEPS) train_op = state_ops.assign_add(completed_sweeps, 1) hook.begin() diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index 6a67c6295d..f4ac70eb1a 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -377,11 +377,6 @@ py_test( size = "large", srcs = ["python/training/shampoo_test.py"], srcs_version = "PY2AND3", - tags = [ - "noasan", # b/116875897 - "nomsan", - "notsan", - ], deps = [ ":opt_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index cb1f707028..c230919168 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -159,12 +159,7 @@ py_test( ], shard_count = 4, srcs_version = "PY2AND3", - tags = [ - "no_pip_gpu", # b/63391119 - "noasan", # b/116875897 - "nomsan", - "notsan", - ], + tags = ["no_pip_gpu"], # b/63391119 deps = [ ":estimators", ":feature_keys", -- GitLab From ce1cdd52eda4b40ff8fb8c09bc178210883b3773 Mon Sep 17 00:00:00 2001 From: Russell Power Date: Mon, 1 Oct 2018 10:57:32 -0700 Subject: [PATCH 0153/1085] Make GCS filesystem/metadata lookup retries configurable PiperOrigin-RevId: 215243030 --- .../cloud/compute_engine_metadata_client.cc | 15 +- .../cloud/compute_engine_metadata_client.h | 10 +- .../compute_engine_metadata_client_test.cc | 6 +- .../compute_engine_zone_provider_test.cc | 8 +- .../core/platform/cloud/gcs_file_system.cc | 25 +- .../core/platform/cloud/gcs_file_system.h | 7 +- .../platform/cloud/gcs_file_system_test.cc | 1286 +++++++++-------- .../cloud/google_auth_provider_test.cc | 20 +- .../platform/cloud/retrying_file_system.h | 67 +- .../cloud/retrying_file_system_test.cc | 102 +- .../core/platform/cloud/retrying_utils.cc | 35 +- .../core/platform/cloud/retrying_utils.h | 29 +- .../platform/cloud/retrying_utils_test.cc | 32 +- 13 files changed, 849 insertions(+), 793 deletions(-) diff --git a/tensorflow/core/platform/cloud/compute_engine_metadata_client.cc b/tensorflow/core/platform/cloud/compute_engine_metadata_client.cc index f41b83ac34..affb68ebbb 100644 --- a/tensorflow/core/platform/cloud/compute_engine_metadata_client.cc +++ b/tensorflow/core/platform/cloud/compute_engine_metadata_client.cc @@ -17,7 +17,6 @@ limitations under the License. #include #include "tensorflow/core/platform/cloud/curl_http_request.h" -#include "tensorflow/core/platform/cloud/retrying_utils.h" namespace tensorflow { @@ -25,21 +24,14 @@ namespace { // The URL to retrieve metadata when running in Google Compute Engine. constexpr char kGceMetadataBaseUrl[] = "http://metadata/computeMetadata/v1/"; -// The default initial delay between retries with exponential backoff. -constexpr int kInitialRetryDelayUsec = 500000; // 0.5 sec } // namespace -ComputeEngineMetadataClient::ComputeEngineMetadataClient( - std::shared_ptr http_request_factory) - : ComputeEngineMetadataClient(std::move(http_request_factory), - kInitialRetryDelayUsec) {} - ComputeEngineMetadataClient::ComputeEngineMetadataClient( std::shared_ptr http_request_factory, - int64 initial_retry_delay_usec) + const RetryConfig& config) : http_request_factory_(std::move(http_request_factory)), - initial_retry_delay_usec_(initial_retry_delay_usec) {} + retry_config_(config) {} Status ComputeEngineMetadataClient::GetMetadata( const string& path, std::vector* response_buffer) { @@ -52,8 +44,7 @@ Status ComputeEngineMetadataClient::GetMetadata( return Status::OK(); }; - return RetryingUtils::CallWithRetries(get_metadata_from_gce, - initial_retry_delay_usec_); + return RetryingUtils::CallWithRetries(get_metadata_from_gce, retry_config_); } } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/compute_engine_metadata_client.h b/tensorflow/core/platform/cloud/compute_engine_metadata_client.h index 534ccf30b2..7f060327da 100644 --- a/tensorflow/core/platform/cloud/compute_engine_metadata_client.h +++ b/tensorflow/core/platform/cloud/compute_engine_metadata_client.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/cloud/http_request.h" +#include "tensorflow/core/platform/cloud/retrying_utils.h" namespace tensorflow { @@ -31,10 +32,11 @@ namespace tensorflow { class ComputeEngineMetadataClient { public: explicit ComputeEngineMetadataClient( - std::shared_ptr http_request_factory); - ComputeEngineMetadataClient( std::shared_ptr http_request_factory, - int64 initial_retry_delay_usec); + const RetryConfig& config = RetryConfig( + 10000, /* init_delay_time_us = 1 ms */ + 1000000 /* max_delay_time_us = 1 s */ + )); virtual ~ComputeEngineMetadataClient() {} /// \brief Get the metadata value for a given attribute of the metadata @@ -54,7 +56,7 @@ class ComputeEngineMetadataClient { private: std::shared_ptr http_request_factory_; - const int64 initial_retry_delay_usec_; + const RetryConfig retry_config_; TF_DISALLOW_COPY_AND_ASSIGN(ComputeEngineMetadataClient); }; diff --git a/tensorflow/core/platform/cloud/compute_engine_metadata_client_test.cc b/tensorflow/core/platform/cloud/compute_engine_metadata_client_test.cc index 4c41ccaa0e..e891b4a5e9 100644 --- a/tensorflow/core/platform/cloud/compute_engine_metadata_client_test.cc +++ b/tensorflow/core/platform/cloud/compute_engine_metadata_client_test.cc @@ -30,7 +30,8 @@ TEST(ComputeEngineMetadataClientTest, GetMetadata) { std::shared_ptr http_factory = std::make_shared(&requests); - ComputeEngineMetadataClient client(http_factory, 0); + ComputeEngineMetadataClient client(http_factory, + RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK( @@ -56,7 +57,8 @@ TEST(ComputeEngineMetadataClientTest, RetryOnFailure) { std::shared_ptr http_factory = std::make_shared(&requests); - ComputeEngineMetadataClient client(http_factory, 0); + ComputeEngineMetadataClient client(http_factory, + RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK( diff --git a/tensorflow/core/platform/cloud/compute_engine_zone_provider_test.cc b/tensorflow/core/platform/cloud/compute_engine_zone_provider_test.cc index f7477eca23..476e4f9c1f 100644 --- a/tensorflow/core/platform/cloud/compute_engine_zone_provider_test.cc +++ b/tensorflow/core/platform/cloud/compute_engine_zone_provider_test.cc @@ -34,8 +34,8 @@ TEST_F(ComputeEngineZoneProviderTest, GetZone) { auto httpRequestFactory = std::make_shared(&requests); - auto metadata_client = - std::make_shared(httpRequestFactory, 0); + auto metadata_client = std::make_shared( + httpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); ComputeEngineZoneProvider provider(metadata_client); @@ -55,8 +55,8 @@ TEST_F(ComputeEngineZoneProviderTest, InvalidZoneString) { auto httpRequestFactory = std::make_shared(&requests); - auto metadata_client = - std::make_shared(httpRequestFactory, 0); + auto metadata_client = std::make_shared( + httpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); ComputeEngineZoneProvider provider(metadata_client); diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 83ea8539ed..c61b68aeeb 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -333,14 +333,14 @@ class GcsWritableFile : public WritableFile { GcsFileSystem* filesystem, GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, - int64 initial_retry_delay_usec) + RetryConfig retry_config) : bucket_(bucket), object_(object), filesystem_(filesystem), timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), - initial_retry_delay_usec_(initial_retry_delay_usec) { + retry_config_(retry_config) { // TODO: to make it safer, outfile_ should be constructed from an FD if (GetTmpFilename(&tmp_content_filename_).ok()) { outfile_.open(tmp_content_filename_, @@ -357,14 +357,14 @@ class GcsWritableFile : public WritableFile { GcsFileSystem* filesystem, const string& tmp_content_filename, GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, - int64 initial_retry_delay_usec) + RetryConfig retry_config) : bucket_(bucket), object_(object), filesystem_(filesystem), timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), - initial_retry_delay_usec_(initial_retry_delay_usec) { + retry_config_(retry_config) { tmp_content_filename_ = tmp_content_filename; outfile_.open(tmp_content_filename_, std::ofstream::binary | std::ofstream::app); @@ -441,7 +441,7 @@ class GcsWritableFile : public WritableFile { first_attempt = false; return UploadToSession(session_uri, already_uploaded); }, - initial_retry_delay_usec_); + retry_config_); if (upload_status.code() == errors::Code::NOT_FOUND) { // GCS docs recommend retrying the whole upload. We're relying on the // RetryingFileSystem to retry the Sync() call. @@ -586,7 +586,7 @@ class GcsWritableFile : public WritableFile { GcsFileSystem::TimeoutConfig* timeouts_; std::function file_cache_erase_; bool sync_needed_; // whether there is buffered data that needs to be synced - int64 initial_retry_delay_usec_; + RetryConfig retry_config_; }; class GcsReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { @@ -791,7 +791,7 @@ GcsFileSystem::GcsFileSystem( std::unique_ptr zone_provider, size_t block_size, size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age, size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, - size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec, + size_t matching_paths_cache_max_entries, RetryConfig retry_config, TimeoutConfig timeouts, const std::unordered_set& allowed_locations, std::pair* additional_header) : auth_provider_(std::move(auth_provider)), @@ -806,7 +806,7 @@ GcsFileSystem::GcsFileSystem( kCacheNeverExpire, kBucketLocationCacheMaxEntries)), allowed_locations_(allowed_locations), timeouts_(timeouts), - initial_retry_delay_usec_(initial_retry_delay_usec), + retry_config_(retry_config), additional_header_(additional_header) {} Status GcsFileSystem::NewRandomAccessFile( @@ -941,7 +941,7 @@ Status GcsFileSystem::NewWritableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile(bucket, object, this, &timeouts_, [this, fname]() { ClearFileCaches(fname); }, - initial_retry_delay_usec_)); + retry_config_)); return Status::OK(); } @@ -981,7 +981,7 @@ Status GcsFileSystem::NewAppendableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile( bucket, object, this, old_content_filename, &timeouts_, - [this, fname]() { ClearFileCaches(fname); }, initial_retry_delay_usec_)); + [this, fname]() { ClearFileCaches(fname); }, retry_config_)); return Status::OK(); } @@ -1534,7 +1534,7 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) { // on the server side, we can't just retry the whole RenameFile operation // because the source object is already gone. return RetryingUtils::DeleteWithRetries( - [this, &src]() { return DeleteFile(src); }, initial_retry_delay_usec_); + [this, &src]() { return DeleteFile(src); }, retry_config_); } Status GcsFileSystem::IsDirectory(const string& fname) { @@ -1590,8 +1590,7 @@ Status GcsFileSystem::DeleteRecursively(const string& dirname, // and therefore RetryingFileSystem won't pay attention to the failures, // we need to make sure these failures are properly retried. const auto& delete_file_status = RetryingUtils::DeleteWithRetries( - [this, &full_path]() { return DeleteFile(full_path); }, - initial_retry_delay_usec_); + [this, &full_path]() { return DeleteFile(full_path); }, retry_config_); if (!delete_file_status.ok()) { if (IsDirectory(full_path).ok()) { // The object is a directory marker. diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 71db707687..d0840a3046 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -93,7 +93,7 @@ class GcsFileSystem : public FileSystem { uint64 stat_cache_max_age, size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, size_t matching_paths_cache_max_entries, - int64 initial_retry_delay_usec, TimeoutConfig timeouts, + RetryConfig retry_config, TimeoutConfig timeouts, const std::unordered_set& allowed_locations, std::pair* additional_header); @@ -332,7 +332,7 @@ class GcsFileSystem : public FileSystem { GcsStatsInterface* stats_ = nullptr; // Not owned. /// The initial delay for exponential backoffs when retrying failed calls. - const int64 initial_retry_delay_usec_ = 1000000L; + RetryConfig retry_config_; // Additional header material to be transmitted with all GCS requests std::unique_ptr> additional_header_; @@ -344,7 +344,8 @@ class GcsFileSystem : public FileSystem { class RetryingGcsFileSystem : public RetryingFileSystem { public: RetryingGcsFileSystem() - : RetryingFileSystem(std::unique_ptr(new GcsFileSystem)) {} + : RetryingFileSystem(std::unique_ptr(new GcsFileSystem), + RetryConfig(100000 /* init_delay_time_us */)) {} }; } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index 14376ad339..702802b185 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -24,6 +24,8 @@ namespace tensorflow { namespace { static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30); +static RetryConfig kTestRetryConfig(0 /* init_delay_time_us */); + // Default (empty) constraint config static std::unordered_set* kAllowedLocationsDefault = new std::unordered_set(); @@ -62,16 +64,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { "Range: 6-11\n" "Timeouts: 5 1 20\n", "6789")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -108,9 +110,9 @@ TEST(GcsFileSystemTest, 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - *kAllowedLocationsAuto, nullptr /* gcs additional header */); + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -150,9 +152,9 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) { 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - *kAllowedLocationsAuto, nullptr /* gcs additional header */); + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */); std::unique_ptr file; @@ -191,9 +193,9 @@ TEST(GcsFileSystemTest, 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - *kAllowedLocationsAuto, nullptr /* gcs additional header */); + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(tensorflow::errors::FailedPrecondition( @@ -216,16 +218,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { "Range: 3-12\n" "Timeouts: 5 1 20\n", "3456789")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -283,7 +285,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { std::unique_ptr(new FakeZoneProvider), 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -372,7 +374,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { std::unique_ptr(new FakeZoneProvider), 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -414,17 +416,17 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { "Range: 8-15\n" "Timeouts: 5 1 20\n", "89abcdef")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 8 /* block size */, - 16 /* max bytes */, 3600 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 8 /* block size */, 16 /* max bytes */, + 3600 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); char scratch[100]; StringPiece result; // There should only be two HTTP requests issued to GCS even though we iterate @@ -492,7 +494,7 @@ TEST(GcsFileSystemTest, std::unique_ptr(new FakeZoneProvider), 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -513,17 +515,17 @@ TEST(GcsFileSystemTest, TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* read ahead bytes */, 0 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -547,16 +549,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) { "012")}); // Set stat_cache_max_age to 1000s so that StatCache could work. - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 1e3 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Stat the file first so that the file stats are cached. FileStatistics stat; @@ -621,7 +623,7 @@ TEST(GcsFileSystemTest, NewWritableFile) { std::unique_ptr(new FakeZoneProvider), 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -703,16 +705,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { "Timeouts: 5 1 30\n" "Put body: t2\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -773,17 +775,17 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { "Range: 0-7\n" "Timeouts: 5 1 20\n", "01234567")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 8 /* block size */, - 8 /* max bytes */, 3600 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 8 /* block size */, 8 /* max bytes */, + 3600 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Pull the file's first block into the cache. This will trigger the first // HTTP request to GCS. std::unique_ptr rfile; @@ -867,9 +869,9 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 2 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + 0 /* matching paths cache max entries */, + RetryConfig(2 /* .init_delay_time_us */), kTestTimeoutConfig, + *kAllowedLocationsDefault, nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -918,16 +920,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -948,16 +950,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1013,7 +1015,7 @@ TEST(GcsFileSystemTest, NewAppendableFile) { std::unique_ptr(new FakeZoneProvider), 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -1041,16 +1043,16 @@ TEST(GcsFileSystemTest, NewAppendableFile) { TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1075,16 +1077,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { "Range: 0-", content.size() - 1, "\n", "Timeouts: 5 1 20\n"), content)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( @@ -1096,16 +1098,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1120,16 +1122,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); } @@ -1150,16 +1152,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subfolder/\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); } @@ -1176,16 +1178,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"size\": \"100\"}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket1")); TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); @@ -1206,16 +1208,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": []}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(errors::Code::NOT_FOUND, fs.FileExists("gs://bucket/path/file1.txt").code()); @@ -1233,19 +1235,19 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); - EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.FileExists("gs://bucket2/").code()); - EXPECT_EQ(errors::Code::INVALID_ARGUMENT, + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); + EXPECT_EQ(errors::Code::INVALID_ARGUMENT, + fs.FileExists("gs://bucket2/").code()); + EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.FileExists("gs://bucket2").code()); } @@ -1279,7 +1281,7 @@ TEST(GcsFileSystemTest, FileExists_StatCache) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -1306,7 +1308,7 @@ TEST(GcsFileSystemTest, FileExists_DirectoryMark) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -1322,16 +1324,16 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1350,16 +1352,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1379,16 +1381,16 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1407,16 +1409,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1432,16 +1434,16 @@ TEST(GcsFileSystemTest, GetChildren_Root) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); @@ -1457,16 +1459,16 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1498,16 +1500,16 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { " { \"name\": \"path/file4.txt\" }," " { \"name\": \"path/file5.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1525,16 +1527,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK( @@ -1553,16 +1555,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); @@ -1582,16 +1584,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); @@ -1608,16 +1610,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -1634,16 +1636,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); @@ -1652,16 +1654,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1686,16 +1688,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to // any additional HTTP requests to GCS. @@ -1729,16 +1731,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // This loop should trigger the first HTTP request to GCS. for (int i = 0; i < 10; i++) { @@ -1800,7 +1802,7 @@ TEST(GcsFileSystemTest, DeleteFile) { std::unique_ptr(new FakeZoneProvider), 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -1821,16 +1823,16 @@ TEST(GcsFileSystemTest, DeleteFile) { TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.DeleteFile("gs://bucket/").code()); @@ -1871,7 +1873,7 @@ TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) { std::unique_ptr(new FakeZoneProvider), 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -1894,16 +1896,16 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1923,16 +1925,16 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1943,16 +1945,16 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); } @@ -1965,16 +1967,16 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.DeleteDir("gs://bucket/path/").code()); @@ -1988,16 +1990,16 @@ TEST(GcsFileSystemTest, GetFileSize) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); @@ -2006,16 +2008,16 @@ TEST(GcsFileSystemTest, GetFileSize) { TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -2092,16 +2094,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); } @@ -2191,7 +2193,7 @@ TEST(GcsFileSystemTest, RenameFile_Object) { std::unique_ptr(new FakeZoneProvider), 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); // Do an initial read of the source and destination files to load their @@ -2272,7 +2274,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); // Do an initial stat of the destination file to load their contents into the @@ -2332,16 +2334,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK( fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); @@ -2374,16 +2376,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { "Post: yes\n" "Timeouts: 5 1 10\n", "{\"done\": false}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ( errors::Code::UNIMPLEMENTED, @@ -2399,16 +2401,16 @@ TEST(GcsFileSystemTest, Stat_Object) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); @@ -2433,16 +2435,16 @@ TEST(GcsFileSystemTest, Stat_Folder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"subfolder/\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); @@ -2466,16 +2468,16 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); @@ -2487,16 +2489,16 @@ TEST(GcsFileSystemTest, Stat_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); @@ -2511,16 +2513,16 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); @@ -2556,7 +2558,7 @@ TEST(GcsFileSystemTest, Stat_Cache) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); @@ -2598,7 +2600,7 @@ TEST(GcsFileSystemTest, Stat_Cache_Flush) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */); // There should be a single HTTP request to GCS for fs.Stat in this loop. @@ -2628,16 +2630,16 @@ TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"5\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat)); @@ -2660,16 +2662,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -2691,16 +2693,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -2722,16 +2724,16 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": [{\"name\": \"subfolder/\"}]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); @@ -2749,16 +2751,16 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); @@ -2770,16 +2772,16 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); } @@ -2812,16 +2814,16 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { "Timeouts: 5 1 30\n" "Put body: \n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/")); @@ -2839,16 +2841,16 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); TF_EXPECT_OK(fs.CreateDir("gs://bucket")); @@ -2911,16 +2913,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -3004,16 +3006,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -3039,16 +3041,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, @@ -3130,7 +3132,7 @@ TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) { std::unique_ptr(new FakeZoneProvider), 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, add_header /* gcs additional header */); @@ -3199,16 +3201,16 @@ TEST(GcsFileSystemTest, CreateHttpRequest) { "Auth Token: fake_token\n" "Header Hello: world\n", "{}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr request; TF_EXPECT_OK(fs.CreateHttpRequest(&request)); @@ -3262,16 +3264,16 @@ TEST(GcsFileSystemTest, Stat_StatsRecording) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TestGcsStats stats; fs.SetStats(&stats); @@ -3289,16 +3291,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) { "Range: 0-5\n" "Timeouts: 5 1 20\n", "012345")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), 0 /* block size */, - 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TestGcsStats stats; fs.SetStats(&stats); diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc index 07b88a880f..ec31c5ee8c 100644 --- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc +++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc @@ -93,8 +93,8 @@ TEST_F(GoogleAuthProviderTest, EnvironmentVariable_Caching) { std::shared_ptr fakeHttpRequestFactory = std::make_shared(&requests); - auto metadataClient = - std::make_shared(fakeHttpRequestFactory, 0); + auto metadataClient = std::make_shared( + fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); GoogleAuthProvider provider(std::unique_ptr(oauth_client), metadataClient, &env); oauth_client->return_token = "fake-token"; @@ -129,8 +129,8 @@ TEST_F(GoogleAuthProviderTest, GCloudRefreshToken) { FakeEnv env; std::shared_ptr fakeHttpRequestFactory = std::make_shared(&requests); - auto metadataClient = - std::make_shared(fakeHttpRequestFactory, 0); + auto metadataClient = std::make_shared( + fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); GoogleAuthProvider provider(std::unique_ptr(oauth_client), metadataClient, &env); @@ -178,8 +178,8 @@ TEST_F(GoogleAuthProviderTest, RunningOnGCE) { FakeEnv env; std::shared_ptr fakeHttpRequestFactory = std::make_shared(&requests); - auto metadataClient = - std::make_shared(fakeHttpRequestFactory, 0); + auto metadataClient = std::make_shared( + fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); GoogleAuthProvider provider(std::unique_ptr(oauth_client), metadataClient, &env); @@ -206,8 +206,8 @@ TEST_F(GoogleAuthProviderTest, OverrideForTesting) { FakeEnv env; std::shared_ptr fakeHttpRequestFactory = std::make_shared(&empty_requests); - auto metadataClient = - std::make_shared(fakeHttpRequestFactory, 0); + auto metadataClient = std::make_shared( + fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); GoogleAuthProvider provider(std::unique_ptr(oauth_client), metadataClient, &env); @@ -228,8 +228,8 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) { FakeEnv env; std::shared_ptr fakeHttpRequestFactory = std::make_shared(&requests); - auto metadataClient = - std::make_shared(fakeHttpRequestFactory, 0); + auto metadataClient = std::make_shared( + fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */)); GoogleAuthProvider provider(std::unique_ptr(oauth_client), metadataClient, &env); diff --git a/tensorflow/core/platform/cloud/retrying_file_system.h b/tensorflow/core/platform/cloud/retrying_file_system.h index 941ab7ad65..5ce6670dc7 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system.h +++ b/tensorflow/core/platform/cloud/retrying_file_system.h @@ -34,9 +34,9 @@ template class RetryingFileSystem : public FileSystem { public: RetryingFileSystem(std::unique_ptr base_file_system, - int64 delay_microseconds = 1000000) + const RetryConfig& retry_config) : base_file_system_(std::move(base_file_system)), - initial_delay_microseconds_(delay_microseconds) {} + retry_config_(retry_config) {} Status NewRandomAccessFile( const string& filename, @@ -55,7 +55,7 @@ class RetryingFileSystem : public FileSystem { Status FileExists(const string& fname) override { return RetryingUtils::CallWithRetries( [this, &fname]() { return base_file_system_->FileExists(fname); }, - initial_delay_microseconds_); + retry_config_); } Status GetChildren(const string& dir, std::vector* result) override { @@ -63,7 +63,7 @@ class RetryingFileSystem : public FileSystem { [this, &dir, result]() { return base_file_system_->GetChildren(dir, result); }, - initial_delay_microseconds_); + retry_config_); } Status GetMatchingPaths(const string& pattern, @@ -72,31 +72,31 @@ class RetryingFileSystem : public FileSystem { [this, &pattern, result]() { return base_file_system_->GetMatchingPaths(pattern, result); }, - initial_delay_microseconds_); + retry_config_); } Status Stat(const string& fname, FileStatistics* stat) override { return RetryingUtils::CallWithRetries( [this, &fname, stat]() { return base_file_system_->Stat(fname, stat); }, - initial_delay_microseconds_); + retry_config_); } Status DeleteFile(const string& fname) override { return RetryingUtils::DeleteWithRetries( [this, &fname]() { return base_file_system_->DeleteFile(fname); }, - initial_delay_microseconds_); + retry_config_); } Status CreateDir(const string& dirname) override { return RetryingUtils::CallWithRetries( [this, &dirname]() { return base_file_system_->CreateDir(dirname); }, - initial_delay_microseconds_); + retry_config_); } Status DeleteDir(const string& dirname) override { return RetryingUtils::DeleteWithRetries( [this, &dirname]() { return base_file_system_->DeleteDir(dirname); }, - initial_delay_microseconds_); + retry_config_); } Status GetFileSize(const string& fname, uint64* file_size) override { @@ -104,7 +104,7 @@ class RetryingFileSystem : public FileSystem { [this, &fname, file_size]() { return base_file_system_->GetFileSize(fname, file_size); }, - initial_delay_microseconds_); + retry_config_); } Status RenameFile(const string& src, const string& target) override { @@ -112,13 +112,13 @@ class RetryingFileSystem : public FileSystem { [this, &src, &target]() { return base_file_system_->RenameFile(src, target); }, - initial_delay_microseconds_); + retry_config_); } Status IsDirectory(const string& dirname) override { return RetryingUtils::CallWithRetries( [this, &dirname]() { return base_file_system_->IsDirectory(dirname); }, - initial_delay_microseconds_); + retry_config_); } Status DeleteRecursively(const string& dirname, int64* undeleted_files, @@ -128,7 +128,7 @@ class RetryingFileSystem : public FileSystem { return base_file_system_->DeleteRecursively(dirname, undeleted_files, undeleted_dirs); }, - initial_delay_microseconds_); + retry_config_); } void FlushCaches() override { base_file_system_->FlushCaches(); } @@ -137,7 +137,7 @@ class RetryingFileSystem : public FileSystem { private: std::unique_ptr base_file_system_; - const int64 initial_delay_microseconds_; + const RetryConfig retry_config_; TF_DISALLOW_COPY_AND_ASSIGN(RetryingFileSystem); }; @@ -147,9 +147,8 @@ namespace retrying_internals { class RetryingRandomAccessFile : public RandomAccessFile { public: RetryingRandomAccessFile(std::unique_ptr base_file, - int64 delay_microseconds) - : base_file_(std::move(base_file)), - initial_delay_microseconds_(delay_microseconds) {} + const RetryConfig& retry_config) + : base_file_(std::move(base_file)), retry_config_(retry_config) {} Status Read(uint64 offset, size_t n, StringPiece* result, char* scratch) const override { @@ -157,20 +156,19 @@ class RetryingRandomAccessFile : public RandomAccessFile { [this, offset, n, result, scratch]() { return base_file_->Read(offset, n, result, scratch); }, - initial_delay_microseconds_); + retry_config_); } private: std::unique_ptr base_file_; - const int64 initial_delay_microseconds_; + const RetryConfig retry_config_; }; class RetryingWritableFile : public WritableFile { public: RetryingWritableFile(std::unique_ptr base_file, - int64 delay_microseconds) - : base_file_(std::move(base_file)), - initial_delay_microseconds_(delay_microseconds) {} + const RetryConfig& retry_config) + : base_file_(std::move(base_file)), retry_config_(retry_config) {} ~RetryingWritableFile() override { // Makes sure the retrying version of Close() is called in the destructor. @@ -179,25 +177,24 @@ class RetryingWritableFile : public WritableFile { Status Append(StringPiece data) override { return RetryingUtils::CallWithRetries( - [this, &data]() { return base_file_->Append(data); }, - initial_delay_microseconds_); + [this, &data]() { return base_file_->Append(data); }, retry_config_); } Status Close() override { return RetryingUtils::CallWithRetries( - [this]() { return base_file_->Close(); }, initial_delay_microseconds_); + [this]() { return base_file_->Close(); }, retry_config_); } Status Flush() override { return RetryingUtils::CallWithRetries( - [this]() { return base_file_->Flush(); }, initial_delay_microseconds_); + [this]() { return base_file_->Flush(); }, retry_config_); } Status Sync() override { return RetryingUtils::CallWithRetries( - [this]() { return base_file_->Sync(); }, initial_delay_microseconds_); + [this]() { return base_file_->Sync(); }, retry_config_); } private: std::unique_ptr base_file_; - const int64 initial_delay_microseconds_; + const RetryConfig retry_config_; }; } // namespace retrying_internals @@ -210,9 +207,9 @@ Status RetryingFileSystem::NewRandomAccessFile( [this, &filename, &base_file]() { return base_file_system_->NewRandomAccessFile(filename, &base_file); }, - initial_delay_microseconds_)); + retry_config_)); result->reset(new retrying_internals::RetryingRandomAccessFile( - std::move(base_file), initial_delay_microseconds_)); + std::move(base_file), retry_config_)); return Status::OK(); } @@ -224,9 +221,9 @@ Status RetryingFileSystem::NewWritableFile( [this, &filename, &base_file]() { return base_file_system_->NewWritableFile(filename, &base_file); }, - initial_delay_microseconds_)); + retry_config_)); result->reset(new retrying_internals::RetryingWritableFile( - std::move(base_file), initial_delay_microseconds_)); + std::move(base_file), retry_config_)); return Status::OK(); } @@ -238,9 +235,9 @@ Status RetryingFileSystem::NewAppendableFile( [this, &filename, &base_file]() { return base_file_system_->NewAppendableFile(filename, &base_file); }, - initial_delay_microseconds_)); + retry_config_)); result->reset(new retrying_internals::RetryingWritableFile( - std::move(base_file), initial_delay_microseconds_)); + std::move(base_file), retry_config_)); return Status::OK(); } @@ -252,7 +249,7 @@ Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile( return base_file_system_->NewReadOnlyMemoryRegionFromFile(filename, result); }, - initial_delay_microseconds_); + retry_config_); } } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index 5910fef1d2..868eea096c 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -184,7 +184,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_ImmediateSuccess) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->random_access_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped random access file. std::unique_ptr random_access_file; @@ -211,7 +212,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_SuccessWith3rdTry) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->random_access_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped random access file. std::unique_ptr random_access_file; @@ -235,7 +237,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->random_access_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped random access file. std::unique_ptr random_access_file; @@ -265,7 +268,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_NoRetriesForSomeErrors) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->random_access_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped random access file. std::unique_ptr random_access_file; @@ -291,7 +295,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_ImmediateSuccess) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->writable_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped writable file. std::unique_ptr writable_file; @@ -317,7 +322,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->writable_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped writable file. std::unique_ptr writable_file; @@ -343,7 +349,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry_ViaDestructor) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->writable_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped writable file. std::unique_ptr writable_file; @@ -368,7 +375,8 @@ TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->writable_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped appendable file. std::unique_ptr writable_file; @@ -391,7 +399,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); base_fs->writable_file_to_return = std::move(base_file); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); // Retrieve the wrapped writable file. std::unique_ptr writable_file; @@ -412,7 +421,8 @@ TEST(RetryingFileSystemTest, std::make_tuple("NewReadOnlyMemoryRegionFromFile", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::unique_ptr result; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result)); @@ -423,7 +433,8 @@ TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) { CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::unique_ptr result; const auto& status = @@ -440,7 +451,8 @@ TEST(RetryingFileSystemTest, GetChildren_SuccessWith2ndTry) { std::make_tuple("GetChildren", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK(fs.GetChildren("gs://path", &result)); @@ -450,7 +462,8 @@ TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; const auto& status = fs.GetChildren("gs://path", &result); @@ -466,7 +479,8 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) { std::make_tuple("GetMatchingPaths", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://path/dir", &result)); @@ -477,7 +491,8 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) { CreateRetriableErrors("GetMatchingPaths", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; const auto& status = fs.GetMatchingPaths("gs://path/dir", &result); @@ -492,7 +507,8 @@ TEST(RetryingFileSystemTest, DeleteFile_SuccessWith2ndTry) { std::make_tuple("DeleteFile", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK(fs.DeleteFile("gs://path/file.txt")); @@ -502,7 +518,8 @@ TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; const auto& status = fs.DeleteFile("gs://path/file.txt"); @@ -517,7 +534,8 @@ TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) { std::make_tuple("CreateDir", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK(fs.CreateDir("gs://path/newdir")); @@ -527,7 +545,8 @@ TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; const auto& status = fs.CreateDir("gs://path/newdir"); @@ -542,7 +561,8 @@ TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) { std::make_tuple("DeleteDir", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; TF_EXPECT_OK(fs.DeleteDir("gs://path/dir")); @@ -552,7 +572,8 @@ TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; const auto& status = fs.DeleteDir("gs://path/dir"); @@ -568,7 +589,8 @@ TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) { std::make_tuple("GetFileSize", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://path/file.txt", &size)); @@ -578,7 +600,8 @@ TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); uint64 size; const auto& status = fs.GetFileSize("gs://path/file.txt", &size); @@ -593,7 +616,8 @@ TEST(RetryingFileSystemTest, RenameFile_SuccessWith2ndTry) { std::make_tuple("RenameFile", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); TF_EXPECT_OK(fs.RenameFile("old_name", "new_name")); } @@ -602,7 +626,8 @@ TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); const auto& status = fs.RenameFile("old_name", "new_name"); EXPECT_TRUE( @@ -616,7 +641,8 @@ TEST(RetryingFileSystemTest, Stat_SuccessWith2ndTry) { std::make_tuple("Stat", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); FileStatistics stat; TF_EXPECT_OK(fs.Stat("file_name", &stat)); @@ -626,7 +652,8 @@ TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); FileStatistics stat; const auto& status = fs.Stat("file_name", &stat); @@ -639,7 +666,8 @@ TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); const auto& status = fs.FileExists("file_name"); EXPECT_TRUE( @@ -653,7 +681,8 @@ TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { std::make_tuple("FileExists", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); TF_EXPECT_OK(fs.FileExists("gs://path/dir")); } @@ -665,7 +694,8 @@ TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { std::make_tuple("IsDirectory", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); TF_EXPECT_OK(fs.IsDirectory("gs://path/dir")); } @@ -674,7 +704,8 @@ TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) { ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); const auto& status = fs.IsDirectory("gs://path/dir"); EXPECT_TRUE( @@ -689,7 +720,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) { std::make_tuple("DeleteRecursively", Status::OK())}); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK( @@ -701,7 +733,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) { CreateRetriableErrors("DeleteRecursively", 11); std::unique_ptr base_fs( new MockFileSystem(expected_fs_calls)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); int64 undeleted_files, undeleted_dirs; const auto& status = @@ -715,7 +748,8 @@ TEST(RetryingFileSystemTest, FlushCaches) { ExpectedCalls none; bool flushed = false; std::unique_ptr base_fs(new MockFileSystem(none, &flushed)); - RetryingFileSystem fs(std::move(base_fs), 0); + RetryingFileSystem fs( + std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); fs.FlushCaches(); EXPECT_TRUE(flushed); } diff --git a/tensorflow/core/platform/cloud/retrying_utils.cc b/tensorflow/core/platform/cloud/retrying_utils.cc index d2df422024..cb0aecdd35 100644 --- a/tensorflow/core/platform/cloud/retrying_utils.cc +++ b/tensorflow/core/platform/cloud/retrying_utils.cc @@ -23,11 +23,6 @@ namespace tensorflow { namespace { -// In case of failure, every call will be retried kMaxRetries times. -constexpr int kMaxRetries = 10; -// Maximum backoff time in microseconds. -constexpr int64 kMaximumBackoffMicroseconds = 32000000; // 32 seconds. - bool IsRetriable(error::Code code) { switch (code) { case error::UNAVAILABLE: @@ -43,40 +38,41 @@ bool IsRetriable(error::Code code) { } // namespace Status RetryingUtils::CallWithRetries(const std::function& f, - const int64 initial_delay_microseconds) { - return CallWithRetries(f, initial_delay_microseconds, [](int64 micros) { - return Env::Default()->SleepForMicroseconds(micros); - }); + const RetryConfig& config) { + return CallWithRetries( + f, + [](int64 micros) { return Env::Default()->SleepForMicroseconds(micros); }, + config); } Status RetryingUtils::CallWithRetries( - const std::function& f, const int64 initial_delay_microseconds, - const std::function& sleep_usec) { + const std::function& f, + const std::function& sleep_usec, const RetryConfig& config) { int retries = 0; while (true) { auto status = f(); if (!IsRetriable(status.code())) { return status; } - if (retries >= kMaxRetries) { + if (retries >= config.max_retries) { // Return AbortedError, so that it doesn't get retried again somewhere // at a higher level. return Status( error::ABORTED, strings::StrCat( - "All ", kMaxRetries, + "All ", config.max_retries, " retry attempts failed. The last failure: ", status.ToString())); } int64 delay_micros = 0; - if (initial_delay_microseconds > 0) { + if (config.init_delay_time_us > 0) { const int64 random_micros = random::New64() % 1000000; - delay_micros = std::min(initial_delay_microseconds << retries, - kMaximumBackoffMicroseconds) + + delay_micros = std::min(config.init_delay_time_us << retries, + config.max_delay_time_us) + random_micros; } LOG(INFO) << "The operation failed and will be automatically retried in " << (delay_micros / 1000000.0) << " seconds (attempt " - << (retries + 1) << " out of " << kMaxRetries + << (retries + 1) << " out of " << config.max_retries << "), caused by: " << status.ToString(); sleep_usec(delay_micros); retries++; @@ -84,8 +80,7 @@ Status RetryingUtils::CallWithRetries( } Status RetryingUtils::DeleteWithRetries( - const std::function& delete_func, - const int64 initial_delay_microseconds) { + const std::function& delete_func, const RetryConfig& config) { bool is_retried = false; return RetryingUtils::CallWithRetries( [delete_func, &is_retried]() { @@ -96,7 +91,7 @@ Status RetryingUtils::DeleteWithRetries( is_retried = true; return status; }, - initial_delay_microseconds); + config); } } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_utils.h b/tensorflow/core/platform/cloud/retrying_utils.h index 546b8d1c4a..1a7ce1b122 100644 --- a/tensorflow/core/platform/cloud/retrying_utils.h +++ b/tensorflow/core/platform/cloud/retrying_utils.h @@ -21,6 +21,26 @@ limitations under the License. namespace tensorflow { +// Default time before reporting failure: ~100 seconds. +struct RetryConfig { + RetryConfig(int64 init_delay_time_us = 100 * 1000, + int64 max_delay_time_us = 32 * 1000 * 1000, + int max_retries = 10) { + this->init_delay_time_us = init_delay_time_us; + this->max_delay_time_us = max_delay_time_us; + this->max_retries = max_retries; + } + + // In case of failure, every call will be retried max_retries times. + int max_retries; + + // Initial backoff time + int64 init_delay_time_us; + + // Maximum backoff time in microseconds. + int64 max_delay_time_us; +}; + class RetryingUtils { public: /// \brief Retries the function in case of failure with exponential backoff. @@ -31,18 +51,19 @@ class RetryingUtils { /// retries. /// If all retries failed, returns the last error status. static Status CallWithRetries(const std::function& f, - const int64 initial_delay_microseconds); + const RetryConfig& config); + /// sleep_usec is a function that sleeps for the given number of microseconds. static Status CallWithRetries(const std::function& f, - const int64 initial_delay_microseconds, - const std::function& sleep_usec); + const std::function& sleep_usec, + const RetryConfig& config); /// \brief A retrying wrapper for a function that deletes a resource. /// /// The function takes care of the scenario when a delete operation /// returns a failure but succeeds under the hood: if a retry returns /// NOT_FOUND, the whole operation is considered a success. static Status DeleteWithRetries(const std::function& delete_func, - const int64 initial_delay_microseconds); + const RetryConfig& config); }; } // namespace tensorflow diff --git a/tensorflow/core/platform/cloud/retrying_utils_test.cc b/tensorflow/core/platform/cloud/retrying_utils_test.cc index 1b6527618a..75fe8a98f4 100644 --- a/tensorflow/core/platform/cloud/retrying_utils_test.cc +++ b/tensorflow/core/platform/cloud/retrying_utils_test.cc @@ -30,7 +30,8 @@ TEST(RetryingUtilsTest, CallWithRetries_RetryDelays) { }; std::function f = []() { return errors::Unavailable("Failed."); }; - const auto& status = RetryingUtils::CallWithRetries(f, 500000L, sleep); + const auto& status = RetryingUtils::CallWithRetries( + f, sleep, RetryConfig(500000 /* init_delay_time_us */)); EXPECT_EQ(errors::Code::ABORTED, status.code()); EXPECT_TRUE(str_util::StrContains( status.error_message(), @@ -60,8 +61,10 @@ TEST(RetryingUtilsTest, CallWithRetries_NotFoundIsNotRetried) { results.erase(results.begin()); return result; }; - EXPECT_EQ(errors::Code::NOT_FOUND, - RetryingUtils::CallWithRetries(f, 0).code()); + EXPECT_EQ( + errors::Code::NOT_FOUND, + RetryingUtils::CallWithRetries(f, RetryConfig(0 /* init_delay_time_us */)) + .code()); } TEST(RetryingUtilsTest, CallWithRetries_ImmediateSuccess) { @@ -74,7 +77,8 @@ TEST(RetryingUtilsTest, CallWithRetries_ImmediateSuccess) { results.erase(results.begin()); return result; }; - TF_EXPECT_OK(RetryingUtils::CallWithRetries(f, 1.0, sleep)); + TF_EXPECT_OK(RetryingUtils::CallWithRetries( + f, sleep, RetryConfig(1L /* init_delay_time_us */))); } TEST(RetryingUtilsTest, CallWithRetries_EventualSuccess) { @@ -86,7 +90,8 @@ TEST(RetryingUtilsTest, CallWithRetries_EventualSuccess) { results.erase(results.begin()); return result; }; - TF_EXPECT_OK(RetryingUtils::CallWithRetries(f, 0)); + TF_EXPECT_OK(RetryingUtils::CallWithRetries( + f, RetryConfig(0 /* init_delay_time_us */))); } TEST(RetryingUtilsTest, DeleteWithRetries_ImmediateSuccess) { @@ -96,7 +101,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_ImmediateSuccess) { delete_results.erase(delete_results.begin()); return result; }; - TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0)); + TF_EXPECT_OK(RetryingUtils::DeleteWithRetries( + delete_func, RetryConfig(0 /* init_delay_time_us */))); } TEST(RetryingUtilsTest, DeleteWithRetries_EventualSuccess) { @@ -106,7 +112,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_EventualSuccess) { delete_results.erase(delete_results.begin()); return result; }; - TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0)); + TF_EXPECT_OK(RetryingUtils::DeleteWithRetries( + delete_func, RetryConfig(0 /* init_delay_time_us */))); } TEST(RetryingUtilsTest, DeleteWithRetries_PermissionDeniedNotRetried) { @@ -118,7 +125,9 @@ TEST(RetryingUtilsTest, DeleteWithRetries_PermissionDeniedNotRetried) { return result; }; EXPECT_EQ(errors::Code::PERMISSION_DENIED, - RetryingUtils::DeleteWithRetries(delete_func, 0).code()); + RetryingUtils::DeleteWithRetries( + delete_func, RetryConfig(0 /* init_delay_time_us */)) + .code()); } TEST(RetryingUtilsTest, DeleteWithRetries_SuccessThroughFileNotFound) { @@ -129,7 +138,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_SuccessThroughFileNotFound) { delete_results.erase(delete_results.begin()); return result; }; - TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0)); + TF_EXPECT_OK(RetryingUtils::DeleteWithRetries( + delete_func, RetryConfig(0 /* init_delay_time_us */))); } TEST(RetryingUtilsTest, DeleteWithRetries_FirstNotFoundReturnedAsIs) { @@ -140,7 +150,9 @@ TEST(RetryingUtilsTest, DeleteWithRetries_FirstNotFoundReturnedAsIs) { return result; }; EXPECT_EQ(error::NOT_FOUND, - RetryingUtils::DeleteWithRetries(delete_func, 0).code()); + RetryingUtils::DeleteWithRetries( + delete_func, RetryConfig(0 /* init_delay_time_us */)) + .code()); } } // namespace -- GitLab From 84a051e7d0cd1406c1bb846efc677c8aa3fc896e Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 1 Oct 2018 11:12:03 -0700 Subject: [PATCH 0154/1085] Fix typo. PiperOrigin-RevId: 215246174 --- tensorflow/python/autograph/CONTRIBUTING.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/python/autograph/CONTRIBUTING.md b/tensorflow/python/autograph/CONTRIBUTING.md index 1ded5ba5f6..f3587a4384 100644 --- a/tensorflow/python/autograph/CONTRIBUTING.md +++ b/tensorflow/python/autograph/CONTRIBUTING.md @@ -9,8 +9,6 @@ In preparation for TF 2.0, we moved the code base of AutoGraph from does not impact functionality, and AutoGraph will remain accessible under `tensorflow.contrib.autograph` until `tensorflow.contrib` is retired. -When - ## TensorFlow Code of Conduct Please review and follow the [TensorFlow Code of Conduct](../../CODE_OF_CONDUCT.md). -- GitLab From 2bbf05148ad94928c1c828d40e479afdf34e2ef8 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 1 Oct 2018 11:24:41 -0700 Subject: [PATCH 0155/1085] Automated rollback of commit 6a787235b95dd3040fc5ff7fb7104585e746c66a PiperOrigin-RevId: 215248737 --- tensorflow/core/kernels/batching_util/BUILD | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tensorflow/core/kernels/batching_util/BUILD b/tensorflow/core/kernels/batching_util/BUILD index 039b0db144..0d53240330 100644 --- a/tensorflow/core/kernels/batching_util/BUILD +++ b/tensorflow/core/kernels/batching_util/BUILD @@ -12,11 +12,6 @@ cc_library( name = "periodic_function_dynamic", srcs = ["periodic_function.cc"], hdrs = ["periodic_function.h"], - visibility = [ - "//learning/serving:__subpackages__", - "//tensorflow:internal", - "//tensorflow_serving:__subpackages__", - ], deps = [ "//tensorflow/core:framework_headers_lib", "//tensorflow/core:protos_all_cc", @@ -25,11 +20,6 @@ cc_library( cc_library( name = "periodic_function", - visibility = [ - "//learning/serving:__subpackages__", - "//tensorflow:internal", - "//tensorflow_serving:__subpackages__", - ], deps = [ ":periodic_function_dynamic", "//tensorflow/core:lib", @@ -198,11 +188,6 @@ cc_library( testonly = 1, srcs = ["fake_clock_env.cc"], hdrs = ["fake_clock_env.h"], - visibility = [ - "//learning/serving:__subpackages__", - "//tensorflow:internal", - "//tensorflow_serving:__subpackages__", - ], deps = [ "//tensorflow/core:lib", "//tensorflow/core:tensorflow", -- GitLab From a9b01e8a31a02188bc81349c103f136095f322ac Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 11:26:02 -0700 Subject: [PATCH 0156/1085] internal change only PiperOrigin-RevId: 215248985 --- tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc index 8e6e9aa0cd..1c5ea2d997 100644 --- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc +++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc @@ -237,7 +237,8 @@ void StartMonitoring(const tensorflow::string& service_addr, int duration_ms, MonitorResponse response; TF_QCHECK_OK(FromGrpcStatus(stub->Monitor(&context, request, &response))); - std::cout << "Xprof Monitoring Results (Sample " << query + 1 << "):\n\n" + std::cout << "Cloud TPU Monitoring Results (Sample " << query + 1 + << "):\n\n" << response.data() << std::flush; } } -- GitLab From f0f301f05fb1f1965c966ef57cc390e48d966f12 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 1 Oct 2018 11:29:30 -0700 Subject: [PATCH 0157/1085] Add deprecation notice for BasicRNNCell, which will be replaced by keras.SimpleRNNCell. PiperOrigin-RevId: 215249611 --- tensorflow/python/kernel_tests/rnn_test.py | 39 ++++ tensorflow/python/ops/rnn_cell_impl.py | 4 +- ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt | 202 ------------------ .../golden/v2/tensorflow.nn.rnn_cell.pbtxt | 4 - 4 files changed, 42 insertions(+), 207 deletions(-) delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py index 05ad9f6336..2f6963f6b8 100644 --- a/tensorflow/python/kernel_tests/rnn_test.py +++ b/tensorflow/python/kernel_tests/rnn_test.py @@ -535,6 +535,45 @@ class RNNTest(test.TestCase): self.assertAllClose(tf_out, k_out) self.assertAllClose(tf_state, k_state) + def testSimpleRNNCellAndBasicRNNCellComparison(self): + input_shape = 10 + output_shape = 5 + timestep = 4 + batch = 20 + (x_train, _), _ = testing_utils.get_test_data( + train_samples=batch, + test_samples=0, + input_shape=(timestep, input_shape), + num_classes=output_shape) + fix_weights_generator = keras.layers.SimpleRNNCell(output_shape) + fix_weights_generator.build((None, input_shape)) + # The SimpleRNNCell contains 3 weights: kernel, recurrent_kernel, and bias + # The BasicRNNCell contains 2 weight: kernel and bias, where kernel is + # zipped [kernel, recurrent_kernel] in SimpleRNNCell. + keras_weights = fix_weights_generator.get_weights() + kernel, recurrent_kernel, bias = keras_weights + tf_weights = [np.concatenate((kernel, recurrent_kernel)), bias] + + with self.test_session(graph=ops_lib.Graph()) as sess: + inputs = array_ops.placeholder( + dtypes.float32, shape=(None, timestep, input_shape)) + cell = keras.layers.SimpleRNNCell(output_shape) + k_out, k_state = rnn.dynamic_rnn( + cell, inputs, dtype=dtypes.float32) + cell.set_weights(keras_weights) + [k_out, k_state] = sess.run([k_out, k_state], {inputs: x_train}) + with self.test_session(graph=ops_lib.Graph()) as sess: + inputs = array_ops.placeholder( + dtypes.float32, shape=(None, timestep, input_shape)) + cell = rnn_cell_impl.BasicRNNCell(output_shape) + tf_out, tf_state = rnn.dynamic_rnn( + cell, inputs, dtype=dtypes.float32) + cell.set_weights(tf_weights) + [tf_out, tf_state] = sess.run([tf_out, tf_state], {inputs: x_train}) + + self.assertAllClose(tf_out, k_out) + self.assertAllClose(tf_state, k_state) + def testBasicLSTMCellInterchangeWithLSTMCell(self): with self.session(graph=ops_lib.Graph()) as sess: basic_cell = rnn_cell_impl.BasicLSTMCell(1) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index c2751e529a..dd4f3d7a99 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -370,7 +370,7 @@ class LayerRNNCell(RNNCell): *args, **kwargs) -@tf_export("nn.rnn_cell.BasicRNNCell") +@tf_export(v1=["nn.rnn_cell.BasicRNNCell"]) class BasicRNNCell(LayerRNNCell): """The most basic RNN cell. @@ -393,6 +393,8 @@ class BasicRNNCell(LayerRNNCell): `trainable` etc when constructing the cell from configs of get_config(). """ + @deprecated(None, "This class is equivalent as tf.keras.layers.SimpleRNNCell," + " and will be replaced by that in Tensorflow 2.0.") def __init__(self, num_units, activation=None, diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt deleted file mode 100644 index a4483fefa2..0000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt +++ /dev/null @@ -1,202 +0,0 @@ -path: "tensorflow.nn.rnn_cell.BasicRNNCell" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "activity_regularizer" - mtype: "" - } - member { - name: "dtype" - mtype: "" - } - member { - name: "graph" - mtype: "" - } - member { - name: "inbound_nodes" - mtype: "" - } - member { - name: "input" - mtype: "" - } - member { - name: "input_mask" - mtype: "" - } - member { - name: "input_shape" - mtype: "" - } - member { - name: "losses" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member { - name: "non_trainable_variables" - mtype: "" - } - member { - name: "non_trainable_weights" - mtype: "" - } - member { - name: "outbound_nodes" - mtype: "" - } - member { - name: "output" - mtype: "" - } - member { - name: "output_mask" - mtype: "" - } - member { - name: "output_shape" - mtype: "" - } - member { - name: "output_size" - mtype: "" - } - member { - name: "scope_name" - mtype: "" - } - member { - name: "state_size" - mtype: "" - } - member { - name: "trainable_variables" - mtype: "" - } - member { - name: "trainable_weights" - mtype: "" - } - member { - name: "updates" - mtype: "" - } - member { - name: "variables" - mtype: "" - } - member { - name: "weights" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'num_units\', \'activation\', \'reuse\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "add_loss" - argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "add_update" - argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "add_variable" - argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\', \'None\'], " - } - member_method { - name: "apply" - argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "build" - argspec: "args=[\'instance\', \'input_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "call" - argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "compute_mask" - argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "compute_output_shape" - argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "count_params" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "from_config" - argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_config" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_initial_state" - argspec: "args=[\'self\', \'inputs\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } - member_method { - name: "get_input_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_input_mask_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_input_shape_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_losses_for" - argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_output_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_output_mask_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_output_shape_at" - argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_updates_for" - argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_weights" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "set_weights" - argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "zero_state" - argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt index 64697e8a02..24767e250f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt @@ -4,10 +4,6 @@ tf_module { name: "BasicLSTMCell" mtype: "" } - member { - name: "BasicRNNCell" - mtype: "" - } member { name: "DeviceWrapper" mtype: "" -- GitLab From 7cabc6be4e32dfb7f42c7f5e33549984bfdb68a3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 11:44:17 -0700 Subject: [PATCH 0158/1085] Allow zero number of inputs in XRT execute operation. PiperOrigin-RevId: 215252408 --- tensorflow/compiler/xrt/ops/xrt_execute_op.cc | 2 +- tensorflow/compiler/xrt/tests/raw_api_test.cc | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xrt/ops/xrt_execute_op.cc b/tensorflow/compiler/xrt/ops/xrt_execute_op.cc index fda4c31298..40ec1b0ba9 100644 --- a/tensorflow/compiler/xrt/ops/xrt_execute_op.cc +++ b/tensorflow/compiler/xrt/ops/xrt_execute_op.cc @@ -21,7 +21,7 @@ limitations under the License. namespace tensorflow { REGISTER_OP("XRTExecute") - .Attr("Ninputs: int") + .Attr("Ninputs: int >= 0") .Input("computation_handle: int64") .Input("execution_config: string") .Input("input_handles: Ninputs * int64") diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc index 2952feb16a..f590fbf0d9 100644 --- a/tensorflow/compiler/xrt/tests/raw_api_test.cc +++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc @@ -108,6 +108,14 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a, return equal; } +xla::XlaComputation OnePlusTwo() { + xla::XlaBuilder builder("OnePlusTwo"); + auto c0 = xla::ConstantR0(&builder, 1.0f); + auto c1 = xla::ConstantR0(&builder, 2.0f); + xla::Add(c0, c1); + return builder.Build().ValueOrDie(); +} + xla::XlaComputation AddAndScale() { xla::XlaBuilder builder("AddAndScale"); auto p0 = xla::Parameter(&builder, 0, @@ -346,6 +354,39 @@ TEST(RawApiTest, CompileAndExecute) { EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); } +TEST(RawApiTest, CompileAndExecuteZeroArg) { + xrt::XLAComputation c; + auto config = c.mutable_config(); + auto shapes = config->mutable_program_shape(); + *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {}); + + xrt::XRTExecutionConfig e; + e.set_release_input_handles(true); + e.set_release_compilation_handle(true); + StoreComputationSnapshot(OnePlusTwo(), c.mutable_hlo_snapshot()); + + Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag()); + auto e_config = + ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString()); + auto computation = + ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString()); + auto c_handle = ops::XRTCompile(root, computation); + auto result = ops::XRTExecute(root, c_handle, e_config, + std::initializer_list({})); + auto read_back = ops::XRTReadLiteralAndRelease(root, result); + TF_ASSERT_OK(root.status()); + + ClientSession session(root); + std::vector outputs; + TF_EXPECT_OK(session.Run({read_back}, &outputs)); + + xla::LiteralProto response; + EXPECT_TRUE(response.ParseFromString(outputs[0].scalar()())); + + auto expected = xla::LiteralUtil::CreateR0(3.0f); + EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); +} + TEST(RawApiTest, CompileAndExecuteReturnTuple) { xrt::XLAAllocation p0; p0.set_device_ordinal(0); -- GitLab From f1fd53748b99532b2572b8909efcd4f5c06ce28d Mon Sep 17 00:00:00 2001 From: Anna R Date: Mon, 1 Oct 2018 11:53:27 -0700 Subject: [PATCH 0159/1085] Updating function and class tf_export decorators for endpoints according to https://github.com/tensorflow/community/pull/16. In addition to the changes in the doc, I made the following updates (these changes make sense to me and I didn't notice them when compiling the doc): * deprecate saved_model.builder.SavedModelBuilder - replaced with saved_model.SavedModelBuilder * deprecate python_io.tf_record_iterator - replaced with io.tf_record_iterator * deprecate python_io.TFRecordWriter - replaced with io.TFRecordWriter * move reduce_join to tf.string PiperOrigin-RevId: 215253944 --- tensorflow/python/framework/dtypes.py | 4 +- tensorflow/python/framework/errors_impl.py | 6 +- tensorflow/python/framework/graph_io.py | 2 +- tensorflow/python/framework/importer.py | 2 +- tensorflow/python/framework/random_seed.py | 6 +- tensorflow/python/framework/sparse_tensor.py | 2 +- tensorflow/python/lib/io/tf_record.py | 13 +- tensorflow/python/ops/array_ops.py | 44 ++-- .../python/ops/candidate_sampling_ops.py | 8 +- tensorflow/python/ops/check_ops.py | 63 ++++-- tensorflow/python/ops/clip_ops.py | 8 +- tensorflow/python/ops/confusion_matrix.py | 4 +- tensorflow/python/ops/control_flow_ops.py | 2 +- tensorflow/python/ops/data_flow_ops.py | 17 +- tensorflow/python/ops/init_ops.py | 5 + tensorflow/python/ops/linalg_ops.py | 15 +- tensorflow/python/ops/lookup_ops.py | 2 +- tensorflow/python/ops/manip_ops.py | 4 +- tensorflow/python/ops/math_ops.py | 145 ++++++++------ tensorflow/python/ops/nn_impl.py | 6 +- tensorflow/python/ops/nn_ops.py | 8 +- tensorflow/python/ops/numerics.py | 4 +- tensorflow/python/ops/parsing_ops.py | 18 +- tensorflow/python/ops/random_ops.py | 19 +- tensorflow/python/ops/sparse_ops.py | 107 ++++++---- tensorflow/python/ops/special_math_ops.py | 4 +- tensorflow/python/ops/string_ops.py | 7 +- tensorflow/python/saved_model/builder_impl.py | 7 +- tensorflow/python/saved_model/loader_impl.py | 8 +- tensorflow/python/saved_model/main_op_impl.py | 5 +- .../saved_model/signature_def_utils_impl.py | 27 ++- tensorflow/python/saved_model/utils_impl.py | 10 +- .../tools/api/generator/api_init_files.bzl | 1 + .../tools/api/generator/api_init_files_v1.bzl | 1 + tensorflow/python/training/input.py | 3 +- .../api/golden/v1/tensorflow.debugging.pbtxt | 96 +++++++++ .../golden/v1/tensorflow.dtypes.-d-type.pbtxt | 77 +++++++ .../api/golden/v1/tensorflow.dtypes.pbtxt | 20 ++ .../api/golden/v1/tensorflow.graph_util.pbtxt | 4 + .../api/golden/v1/tensorflow.image.pbtxt | 4 + .../golden/v1/tensorflow.initializers.pbtxt | 4 + .../v1/tensorflow.io.-fixed-len-feature.pbtxt | 27 +++ ...rflow.io.-fixed-len-sequence-feature.pbtxt | 31 +++ ...tensorflow.io.-padding-f-i-f-o-queue.pbtxt | 66 ++++++ .../v1/tensorflow.io.-priority-queue.pbtxt | 66 ++++++ .../golden/v1/tensorflow.io.-queue-base.pbtxt | 65 ++++++ .../tensorflow.io.-random-shuffle-queue.pbtxt | 66 ++++++ .../v1/tensorflow.io.-sparse-feature.pbtxt | 35 ++++ ...flow.io.-t-f-record-compression-type.pbtxt | 20 ++ .../tensorflow.io.-t-f-record-options.pbtxt | 17 ++ .../v1/tensorflow.io.-t-f-record-writer.pbtxt | 21 ++ .../v1/tensorflow.io.-var-len-feature.pbtxt | 19 ++ .../tools/api/golden/v1/tensorflow.io.pbtxt | 84 ++++++++ .../api/golden/v1/tensorflow.linalg.pbtxt | 12 ++ .../tools/api/golden/v1/tensorflow.math.pbtxt | 188 ++++++++++++++++++ .../tools/api/golden/v1/tensorflow.nn.pbtxt | 12 ++ .../tools/api/golden/v1/tensorflow.pbtxt | 8 + .../golden/v1/tensorflow.quantization.pbtxt | 4 + .../api/golden/v1/tensorflow.random.pbtxt | 47 +++++ .../v1/tensorflow.saved_model.-builder.pbtxt | 21 ++ .../golden/v1/tensorflow.saved_model.pbtxt | 44 ++++ ...arse.-sparse-conditional-accumulator.pbtxt | 46 +++++ .../v1/tensorflow.sparse.-sparse-tensor.pbtxt | 54 +++++ .../api/golden/v1/tensorflow.sparse.pbtxt | 112 +++++++++++ .../api/golden/v1/tensorflow.strings.pbtxt | 4 + .../api/golden/v1/tensorflow.train.pbtxt | 4 + .../api/golden/v2/tensorflow.debugging.pbtxt | 96 +++++++++ .../golden/v2/tensorflow.dtypes.-d-type.pbtxt | 77 +++++++ .../api/golden/v2/tensorflow.dtypes.pbtxt | 20 ++ .../api/golden/v2/tensorflow.graph_util.pbtxt | 4 + .../api/golden/v2/tensorflow.image.pbtxt | 4 + .../golden/v2/tensorflow.initializers.pbtxt | 4 + .../v2/tensorflow.io.-fixed-len-feature.pbtxt | 27 +++ ...rflow.io.-fixed-len-sequence-feature.pbtxt | 31 +++ ...tensorflow.io.-padding-f-i-f-o-queue.pbtxt | 66 ++++++ .../v2/tensorflow.io.-priority-queue.pbtxt | 66 ++++++ .../golden/v2/tensorflow.io.-queue-base.pbtxt | 65 ++++++ .../tensorflow.io.-random-shuffle-queue.pbtxt | 66 ++++++ .../v2/tensorflow.io.-sparse-feature.pbtxt | 35 ++++ ...flow.io.-t-f-record-compression-type.pbtxt | 20 ++ .../tensorflow.io.-t-f-record-options.pbtxt | 17 ++ .../v2/tensorflow.io.-t-f-record-writer.pbtxt | 21 ++ .../v2/tensorflow.io.-var-len-feature.pbtxt | 19 ++ .../tools/api/golden/v2/tensorflow.io.pbtxt | 84 ++++++++ .../api/golden/v2/tensorflow.linalg.pbtxt | 12 ++ .../tools/api/golden/v2/tensorflow.math.pbtxt | 188 ++++++++++++++++++ .../tools/api/golden/v2/tensorflow.nn.pbtxt | 12 ++ .../tools/api/golden/v2/tensorflow.pbtxt | 8 + .../golden/v2/tensorflow.quantization.pbtxt | 4 + .../api/golden/v2/tensorflow.random.pbtxt | 47 +++++ .../v2/tensorflow.saved_model.-builder.pbtxt | 21 ++ .../golden/v2/tensorflow.saved_model.pbtxt | 44 ++++ ...arse.-sparse-conditional-accumulator.pbtxt | 46 +++++ .../v2/tensorflow.sparse.-sparse-tensor.pbtxt | 54 +++++ .../api/golden/v2/tensorflow.sparse.pbtxt | 112 +++++++++++ .../api/golden/v2/tensorflow.strings.pbtxt | 4 + .../api/golden/v2/tensorflow.train.pbtxt | 4 + 97 files changed, 2926 insertions(+), 217 deletions(-) create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.dtypes.-d-type.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-sequence-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-padding-f-i-f-o-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-priority-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-queue-base.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-random-shuffle-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-sparse-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-compression-type.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-writer.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.-var-len-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.dtypes.-d-type.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-sequence-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-padding-f-i-f-o-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-priority-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-queue-base.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-random-shuffle-queue.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-sparse-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-compression-type.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-writer.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.-var-len-feature.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.saved_model.-builder.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index c3f70df7d8..64d3b42d89 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -26,7 +26,7 @@ from tensorflow.python.util.tf_export import tf_export _np_bfloat16 = pywrap_tensorflow.TF_bfloat16_type() -@tf_export("DType") +@tf_export("dtypes.DType", "DType") class DType(object): """Represents the type of the elements in a `Tensor`. @@ -658,7 +658,7 @@ _PYTHON_TO_TF = { } -@tf_export("as_dtype") +@tf_export("dtypes.as_dtype", "as_dtype") def as_dtype(type_value): """Converts the given `type_value` to a `DType`. diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py index 5af71f2cfb..8b303fa8a9 100644 --- a/tensorflow/python/framework/errors_impl.py +++ b/tensorflow/python/framework/errors_impl.py @@ -25,11 +25,13 @@ from tensorflow.core.lib.core import error_codes_pb2 from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.framework import c_api_util from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import tf_export -@tf_export("OpError", "errors.OpError") +@tf_export("errors.OpError", "OpError") +@deprecation.deprecated_endpoints("OpError") class OpError(Exception): """A generic error that is raised when TensorFlow execution fails. @@ -72,7 +74,7 @@ class OpError(Exception): or `Recv` op, there will be no corresponding `tf.Operation` object. In that case, this will return `None`, and you should - instead use the `tf.OpError.node_def` to + instead use the `tf.errors.OpError.node_def` to discover information about the op. Returns: diff --git a/tensorflow/python/framework/graph_io.py b/tensorflow/python/framework/graph_io.py index be30b16f5f..47e1344eae 100644 --- a/tensorflow/python/framework/graph_io.py +++ b/tensorflow/python/framework/graph_io.py @@ -27,7 +27,7 @@ from tensorflow.python.lib.io import file_io from tensorflow.python.util.tf_export import tf_export -@tf_export('train.write_graph') +@tf_export('io.write_graph', 'train.write_graph') def write_graph(graph_or_graph_def, logdir, name, as_text=True): """Writes a graph proto to a file. diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index e48e67c8a1..c6595918ae 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -329,7 +329,7 @@ def _SetDefaultAttrValues(node_def, op_def): node_def.attr[key].CopyFrom(attr_def.default_value) -@tf_export('import_graph_def') +@tf_export('graph_util.import_graph_def', 'import_graph_def') @deprecated_args(None, 'Please file an issue at ' 'https://github.com/tensorflow/tensorflow/issues if you depend' ' on this feature.', 'op_dict') diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py index 2f9504889a..6f9f347a99 100644 --- a/tensorflow/python/framework/random_seed.py +++ b/tensorflow/python/framework/random_seed.py @@ -22,6 +22,7 @@ from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -33,7 +34,8 @@ def _truncate_seed(seed): return seed % _MAXINT32 # Truncate to fit into 32-bit integer -@tf_export('get_seed') +@tf_export('random.get_seed', 'get_seed') +@deprecation.deprecated_endpoints('get_seed') def get_seed(op_seed): """Returns the local seeds an operation should use given an op-specific seed. @@ -80,7 +82,7 @@ def get_seed(op_seed): return seeds -@tf_export('set_random_seed') +@tf_export('random.set_random_seed', 'set_random_seed') def set_random_seed(seed): """Sets the graph-level random seed. diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py index d1bdd9b80a..41ef2e11d1 100644 --- a/tensorflow/python/framework/sparse_tensor.py +++ b/tensorflow/python/framework/sparse_tensor.py @@ -33,7 +33,7 @@ _override_helper = ops._override_helper # pylint: enable=protected-access -@tf_export("SparseTensor") +@tf_export("sparse.SparseTensor", "SparseTensor") class SparseTensor(_TensorLike): """Represents a sparse tensor. diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py index cce71a2bab..9ab683d96a 100644 --- a/tensorflow/python/lib/io/tf_record.py +++ b/tensorflow/python/lib/io/tf_record.py @@ -22,10 +22,12 @@ from __future__ import print_function from tensorflow.python import pywrap_tensorflow from tensorflow.python.framework import errors from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export("python_io.TFRecordCompressionType") +@tf_export("io.TFRecordCompressionType", "python_io.TFRecordCompressionType") +@deprecation.deprecated_endpoints("python_io.TFRecordCompressionType") class TFRecordCompressionType(object): """The type of compression for the record.""" NONE = 0 @@ -33,7 +35,8 @@ class TFRecordCompressionType(object): GZIP = 2 -@tf_export("python_io.TFRecordOptions") +@tf_export("io.TFRecordOptions", "python_io.TFRecordOptions") +@deprecation.deprecated_endpoints("python_io.TFRecordOptions") class TFRecordOptions(object): """Options used for manipulating TFRecord files.""" compression_type_map = { @@ -143,7 +146,8 @@ class TFRecordOptions(object): return options -@tf_export("python_io.tf_record_iterator") +@tf_export("io.tf_record_iterator", "python_io.tf_record_iterator") +@deprecation.deprecated_endpoints("python_io.tf_record_iterator") def tf_record_iterator(path, options=None): """An iterator that read the records from a TFRecords file. @@ -175,7 +179,8 @@ def tf_record_iterator(path, options=None): reader.Close() -@tf_export("python_io.TFRecordWriter") +@tf_export("io.TFRecordWriter", "python_io.TFRecordWriter") +@deprecation.deprecated_endpoints("python_io.TFRecordWriter") class TFRecordWriter(object): """A class to write records to a TFRecords file. diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index a7f57e94e3..9f5149d5ac 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1204,7 +1204,8 @@ def boolean_mask(tensor, mask, name="boolean_mask", axis=None): return _apply_mask_1d(tensor, mask, axis) -@tf_export("sparse_mask") +@tf_export("sparse.mask", "sparse_mask") +@deprecation.deprecated_endpoints("sparse_mask") def sparse_mask(a, mask_indices, name=None): """Masks elements of `IndexedSlices`. @@ -1226,7 +1227,7 @@ def sparse_mask(a, mask_indices, name=None): # `b` will be the subset of `a` slices at its second and third indices, so # we want to mask its first and last indices (which are at absolute # indices 12, 45) - b = tf.sparse_mask(a, [12, 45]) + b = tf.sparse.mask(a, [12, 45]) b.indices # [26, 37] tf.shape(b.values) # [2, 10] @@ -1382,7 +1383,7 @@ def transpose(a, perm=None, name="transpose", conjugate=False): [10, 11, 12]]]) # Take the transpose of the matrices in dimension-0 - # (this common operation has a shorthand `matrix_transpose`) + # (this common operation has a shorthand `linalg.transpose`) tf.transpose(x, perm=[0, 2, 1]) # [[[1, 4], # [2, 5], # [3, 6]], @@ -1421,7 +1422,8 @@ def transpose(a, perm=None, name="transpose", conjugate=False): # pylint: disable=invalid-name -@tf_export("matrix_transpose", "linalg.transpose") +@tf_export("linalg.transpose", "matrix_transpose") +@deprecation.deprecated_endpoints("matrix_transpose") def matrix_transpose(a, name="matrix_transpose", conjugate=False): """Transposes last two dimensions of tensor `a`. @@ -1429,19 +1431,19 @@ def matrix_transpose(a, name="matrix_transpose", conjugate=False): ```python x = tf.constant([[1, 2, 3], [4, 5, 6]]) - tf.matrix_transpose(x) # [[1, 4], + tf.linalg.transpose(x) # [[1, 4], # [2, 5], # [3, 6]] x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j, 6 + 6j]]) - tf.matrix_transpose(x, conjugate=True) # [[1 - 1j, 4 - 4j], + tf.linalg.transpose(x, conjugate=True) # [[1 - 1j, 4 - 4j], # [2 - 2j, 5 - 5j], # [3 - 3j, 6 - 6j]] # Matrix with two batch dimensions. # x.shape is [1, 2, 3, 4] - # tf.matrix_transpose(x) is shape [1, 2, 4, 3] + # tf.linalg.transpose(x) is shape [1, 2, 4, 3] ``` Note that `tf.matmul` provides kwargs allowing for transpose of arguments. @@ -1452,14 +1454,14 @@ def matrix_transpose(a, name="matrix_transpose", conjugate=False): tf.matmul(matrix, b, transpose_b=True) # Inefficient! - tf.matmul(matrix, tf.matrix_transpose(b)) + tf.matmul(matrix, tf.linalg.transpose(b)) ``` @compatibility(numpy) In `numpy` transposes are memory-efficient constant time operations as they simply return a new view of the same data with adjusted `strides`. - TensorFlow does not support strides, `matrix_transposes` return a new tensor + TensorFlow does not support strides, `linalg.transposes` return a new tensor with the items permuted. @end_compatibility @@ -1467,7 +1469,7 @@ def matrix_transpose(a, name="matrix_transpose", conjugate=False): a: A `Tensor` with `rank >= 2`. name: A name for the operation (optional). conjugate: Optional bool. Setting it to `True` is mathematically equivalent - to tf.conj(tf.matrix_transpose(input)). + to tf.conj(tf.linalg.transpose(input)). Returns: A transposed batch matrix `Tensor`. @@ -1756,7 +1758,8 @@ def _normalize_sparse_shape(shape, name): return (ops.convert_to_tensor(shape, dtype=dtypes.int64, name=name), rank) -@tf_export("sparse_placeholder") +@tf_export("sparse.placeholder", "sparse_placeholder") +@deprecation.deprecated_endpoints("sparse_placeholder") def sparse_placeholder(dtype, shape=None, name=None): """Inserts a placeholder for a sparse tensor that will be always fed. @@ -1767,8 +1770,8 @@ def sparse_placeholder(dtype, shape=None, name=None): For example: ```python - x = tf.sparse_placeholder(tf.float32) - y = tf.sparse_reduce_sum(x) + x = tf.sparse.placeholder(tf.float32) + y = tf.sparse.reduce_sum(x) with tf.Session() as sess: print(sess.run(y)) # ERROR: will fail because x was not fed. @@ -2250,7 +2253,8 @@ def required_space_to_batch_paddings(input_shape, return result_paddings, result_crops -@tf_export("space_to_batch") +@tf_export("nn.space_to_batch", "space_to_batch") +@deprecation.deprecated_endpoints("space_to_batch") def space_to_batch(input, paddings, block_size, name=None): # pylint: disable=redefined-builtin result = space_to_batch_nd( input, @@ -2264,7 +2268,8 @@ def space_to_batch(input, paddings, block_size, name=None): # pylint: disable=r space_to_batch.__doc__ = gen_array_ops.space_to_batch.__doc__ -@tf_export("space_to_depth") +@tf_export("nn.space_to_depth", "space_to_depth") +@deprecation.deprecated_endpoints("space_to_depth") def space_to_depth(input, block_size, name=None, data_format="NHWC"): # pylint: disable=redefined-builtin return gen_array_ops.space_to_depth(input, block_size, data_format, name=name) @@ -2272,7 +2277,8 @@ def space_to_depth(input, block_size, name=None, data_format="NHWC"): # pylint: space_to_depth.__doc__ = gen_array_ops.space_to_depth.__doc__ -@tf_export("depth_to_space") +@tf_export("nn.depth_to_space", "depth_to_space") +@deprecation.deprecated_endpoints("depth_to_space") def depth_to_space(input, block_size, name=None, data_format="NHWC"): # pylint: disable=redefined-builtin return gen_array_ops.depth_to_space(input, block_size, data_format, name=name) @@ -2747,7 +2753,8 @@ def batch_gather(params, indices, name=None): @tf_export("quantize_v2") @deprecation.deprecated( "2017-10-25", - "`tf.quantize_v2` is deprecated, please use `tf.quantize` instead.") + "`tf.quantize_v2` is deprecated, please use `tf.quantization.quantize` " + "instead.") # pylint: disable=missing-docstring def quantize_v2(input, # pylint: disable=redefined-builtin min_range, max_range, @@ -2769,7 +2776,8 @@ quantize_v2.__doc__ = """Please use `tf.quantize` instead.""" # We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate # tf.quantize_v2 in next version of TensorFlow. -@tf_export("quantize") +@tf_export("quantization.quantize", "quantize") +@deprecation.deprecated_endpoints("quantize") def quantize(input, # pylint: disable=redefined-builtin min_range, max_range, diff --git a/tensorflow/python/ops/candidate_sampling_ops.py b/tensorflow/python/ops/candidate_sampling_ops.py index 9ea1ea9c92..98dde995c9 100644 --- a/tensorflow/python/ops/candidate_sampling_ops.py +++ b/tensorflow/python/ops/candidate_sampling_ops.py @@ -23,10 +23,12 @@ from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops # pylint: disable=unused-import from tensorflow.python.ops import gen_candidate_sampling_ops from tensorflow.python.ops import math_ops # pylint: disable=unused-import +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export('nn.uniform_candidate_sampler') +@tf_export('random.uniform_candidate_sampler', 'nn.uniform_candidate_sampler') +@deprecation.deprecated_endpoints('nn.uniform_candidate_sampler') def uniform_candidate_sampler(true_classes, num_true, num_sampled, unique, range_max, seed=None, name=None): """Samples a set of classes using a uniform base distribution. @@ -82,7 +84,9 @@ def uniform_candidate_sampler(true_classes, num_true, num_sampled, unique, seed2=seed2, name=name) -@tf_export('nn.log_uniform_candidate_sampler') +@tf_export('random.log_uniform_candidate_sampler', + 'nn.log_uniform_candidate_sampler') +@deprecation.deprecated_endpoints('nn.log_uniform_candidate_sampler') def log_uniform_candidate_sampler(true_classes, num_true, num_sampled, unique, range_max, seed=None, name=None): """Samples a set of classes using a log-uniform (Zipfian) base distribution. diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index c3cf6e61f2..d607f1d9fb 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -36,6 +36,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export NUMERIC_TYPES = frozenset( @@ -91,7 +92,8 @@ def _shape_and_dtype_str(tensor): return 'shape=%s dtype=%s' % (tensor.shape, tensor.dtype.name) -@tf_export('assert_proper_iterable') +@tf_export('debugging.assert_proper_iterable', 'assert_proper_iterable') +@deprecation.deprecated_endpoints('assert_proper_iterable') def assert_proper_iterable(values): """Static assert that values is a "proper" iterable. @@ -119,7 +121,8 @@ def assert_proper_iterable(values): 'Expected argument "values" to be iterable. Found: %s' % type(values)) -@tf_export('assert_negative') +@tf_export('debugging.assert_negative', 'assert_negative') +@deprecation.deprecated_endpoints('assert_negative') def assert_negative(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x < 0` holds element-wise. @@ -160,7 +163,8 @@ def assert_negative(x, data=None, summarize=None, message=None, name=None): return assert_less(x, zero, data=data, summarize=summarize) -@tf_export('assert_positive') +@tf_export('debugging.assert_positive', 'assert_positive') +@deprecation.deprecated_endpoints('assert_positive') def assert_positive(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x > 0` holds element-wise. @@ -200,7 +204,8 @@ def assert_positive(x, data=None, summarize=None, message=None, name=None): return assert_less(zero, x, data=data, summarize=summarize) -@tf_export('assert_non_negative') +@tf_export('debugging.assert_non_negative', 'assert_non_negative') +@deprecation.deprecated_endpoints('assert_non_negative') def assert_non_negative(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x >= 0` holds element-wise. @@ -242,7 +247,8 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None): return assert_less_equal(zero, x, data=data, summarize=summarize) -@tf_export('assert_non_positive') +@tf_export('debugging.assert_non_positive', 'assert_non_positive') +@deprecation.deprecated_endpoints('assert_non_positive') def assert_non_positive(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x <= 0` holds element-wise. @@ -284,7 +290,7 @@ def assert_non_positive(x, data=None, summarize=None, message=None, name=None): return assert_less_equal(x, zero, data=data, summarize=summarize) -@tf_export('assert_equal') +@tf_export('debugging.assert_equal', 'assert_equal') def assert_equal(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x == y` holds element-wise. @@ -384,7 +390,8 @@ def assert_equal(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_none_equal') +@tf_export('debugging.assert_none_equal', 'assert_none_equal') +@deprecation.deprecated_endpoints('assert_none_equal') def assert_none_equal( x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x != y` holds for all elements. @@ -435,7 +442,8 @@ def assert_none_equal( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_near') +@tf_export('debugging.assert_near', 'assert_near') +@deprecation.deprecated_endpoints('assert_near') def assert_near( x, y, rtol=None, atol=None, data=None, summarize=None, message=None, name=None): @@ -513,7 +521,7 @@ def assert_near( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_less') +@tf_export('debugging.assert_less', 'assert_less') def assert_less(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x < y` holds element-wise. @@ -561,7 +569,8 @@ def assert_less(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_less_equal') +@tf_export('debugging.assert_less_equal', 'assert_less_equal') +@deprecation.deprecated_endpoints('assert_less_equal') def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x <= y` holds element-wise. @@ -609,7 +618,7 @@ def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_greater') +@tf_export('debugging.assert_greater', 'assert_greater') def assert_greater(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x > y` holds element-wise. @@ -657,7 +666,8 @@ def assert_greater(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_greater_equal') +@tf_export('debugging.assert_greater_equal', 'assert_greater_equal') +@deprecation.deprecated_endpoints('assert_greater_equal') def assert_greater_equal(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x >= y` holds element-wise. @@ -755,7 +765,7 @@ def _assert_rank_condition( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_rank') +@tf_export('debugging.assert_rank', 'assert_rank') def assert_rank(x, rank, data=None, summarize=None, message=None, name=None): """Assert `x` has rank equal to `rank`. @@ -817,7 +827,8 @@ def assert_rank(x, rank, data=None, summarize=None, message=None, name=None): return assert_op -@tf_export('assert_rank_at_least') +@tf_export('debugging.assert_rank_at_least', 'assert_rank_at_least') +@deprecation.deprecated_endpoints('assert_rank_at_least') def assert_rank_at_least( x, rank, data=None, summarize=None, message=None, name=None): """Assert `x` has rank equal to `rank` or higher. @@ -948,7 +959,8 @@ def _assert_ranks_condition( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('assert_rank_in') +@tf_export('debugging.assert_rank_in', 'assert_rank_in') +@deprecation.deprecated_endpoints('assert_rank_in') def assert_rank_in( x, ranks, data=None, summarize=None, message=None, name=None): """Assert `x` has rank in `ranks`. @@ -1010,7 +1022,8 @@ def assert_rank_in( return assert_op -@tf_export('assert_integer') +@tf_export('debugging.assert_integer', 'assert_integer') +@deprecation.deprecated_endpoints('assert_integer') def assert_integer(x, message=None, name=None): """Assert that `x` is of integer dtype. @@ -1048,7 +1061,8 @@ def assert_integer(x, message=None, name=None): return control_flow_ops.no_op('statically_determined_was_integer') -@tf_export('assert_type') +@tf_export('debugging.assert_type', 'assert_type') +@deprecation.deprecated_endpoints('assert_type') def assert_type(tensor, tf_type, message=None, name=None): """Statically asserts that the given `Tensor` is of the specified type. @@ -1095,12 +1109,14 @@ def _get_diff_for_monotonic_comparison(x): return control_flow_ops.cond(is_shorter_than_two, short_result, diff) -@tf_export('is_numeric_tensor') +@tf_export('debugging.is_numeric_tensor', 'is_numeric_tensor') +@deprecation.deprecated_endpoints('is_numeric_tensor') def is_numeric_tensor(tensor): return isinstance(tensor, ops.Tensor) and tensor.dtype in NUMERIC_TYPES -@tf_export('is_non_decreasing') +@tf_export('debugging.is_non_decreasing', 'is_non_decreasing') +@deprecation.deprecated_endpoints('is_non_decreasing') def is_non_decreasing(x, name=None): """Returns `True` if `x` is non-decreasing. @@ -1127,7 +1143,8 @@ def is_non_decreasing(x, name=None): return math_ops.reduce_all(math_ops.less_equal(zero, diff)) -@tf_export('is_strictly_increasing') +@tf_export('debugging.is_strictly_increasing', 'is_strictly_increasing') +@deprecation.deprecated_endpoints('is_strictly_increasing') def is_strictly_increasing(x, name=None): """Returns `True` if `x` is strictly increasing. @@ -1202,7 +1219,8 @@ def _assert_same_base_type(items, expected_type=None): return expected_type -@tf_export('assert_same_float_dtype') +@tf_export('debugging.assert_same_float_dtype', 'assert_same_float_dtype') +@deprecation.deprecated_endpoints('assert_same_float_dtype') def assert_same_float_dtype(tensors=None, dtype=None): """Validate and return float type based on `tensors` and `dtype`. @@ -1231,7 +1249,8 @@ def assert_same_float_dtype(tensors=None, dtype=None): return dtype -@tf_export('assert_scalar') +@tf_export('debugging.assert_scalar', 'assert_scalar') +@deprecation.deprecated_endpoints('assert_scalar') def assert_scalar(tensor, name=None): with ops.name_scope(name, 'assert_scalar', [tensor]) as name_scope: tensor = ops.convert_to_tensor(tensor, name=name_scope) diff --git a/tensorflow/python/ops/clip_ops.py b/tensorflow/python/ops/clip_ops.py index 29468431b3..45516068f4 100644 --- a/tensorflow/python/ops/clip_ops.py +++ b/tensorflow/python/ops/clip_ops.py @@ -30,6 +30,7 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import numerics +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -76,8 +77,8 @@ def clip_by_value(t, clip_value_min, clip_value_max, return t_max # TODO(scottzhu): switch to use new implmentation in 2 weeks. - # return gen_math_ops.clip_by_value( - # t, clip_value_min, clip_value_max, name=name) + # return gen_math_ops.clip_by_value( + # t, clip_value_min, clip_value_max, name=name) # TODO(scottzhu): switch to use new implmentation in 2 weeks. @@ -159,7 +160,8 @@ def clip_by_norm(t, clip_norm, axes=None, name=None): return tclip -@tf_export("global_norm") +@tf_export("linalg.global_norm", "global_norm") +@deprecation.deprecated_endpoints("global_norm") def global_norm(t_list, name=None): """Computes the global norm of multiple tensors. diff --git a/tensorflow/python/ops/confusion_matrix.py b/tensorflow/python/ops/confusion_matrix.py index c09154129f..8259142456 100644 --- a/tensorflow/python/ops/confusion_matrix.py +++ b/tensorflow/python/ops/confusion_matrix.py @@ -26,6 +26,7 @@ from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -89,7 +90,8 @@ def remove_squeezable_dimensions( return labels, predictions -@tf_export('confusion_matrix') +@tf_export('train.confusion_matrix', 'confusion_matrix') +@deprecation.deprecated_endpoints('confusion_matrix') def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32, name=None, weights=None): """Computes the confusion matrix from predictions and labels. diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 9d7d31df22..8ad71fe00c 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -106,7 +106,7 @@ def _summarize_eager(tensor, summarize=None): # Assert and Print are special symbols in python, so we must # use an upper-case version of them. -@tf_export("Assert") +@tf_export("debugging.Assert", "Assert") @tf_should_use.should_use_result def Assert(condition, data, summarize=None, name=None): """Asserts that the given condition is true. diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index 69c0fcbbee..97b6f3bd9c 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -39,6 +39,7 @@ from tensorflow.python.ops import resource_variable_ops # go/tf-wildcard-import # pylint: disable=wildcard-import from tensorflow.python.ops.gen_data_flow_ops import * +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export # pylint: enable=wildcard-import @@ -112,7 +113,8 @@ def _shape_common(s1, s2): # pylint: disable=protected-access -@tf_export("QueueBase") +@tf_export("io.QueueBase", "QueueBase") +@deprecation.deprecated_endpoints("QueueBase") class QueueBase(object): """Base class for queue implementations. @@ -604,7 +606,8 @@ def _shared_name(shared_name): return shared_name -@tf_export("RandomShuffleQueue") +@tf_export("io.RandomShuffleQueue", "RandomShuffleQueue") +@deprecation.deprecated_endpoints("RandomShuffleQueue") class RandomShuffleQueue(QueueBase): """A queue implementation that dequeues elements in a random order. @@ -746,7 +749,8 @@ class FIFOQueue(QueueBase): super(FIFOQueue, self).__init__(dtypes, shapes, names, queue_ref) -@tf_export("PaddingFIFOQueue") +@tf_export("io.PaddingFIFOQueue", "PaddingFIFOQueue") +@deprecation.deprecated_endpoints("PaddingFIFOQueue") class PaddingFIFOQueue(QueueBase): """A FIFOQueue that supports batching variable-sized tensors by padding. @@ -820,7 +824,8 @@ class PaddingFIFOQueue(QueueBase): super(PaddingFIFOQueue, self).__init__(dtypes, shapes, names, queue_ref) -@tf_export("PriorityQueue") +@tf_export("io.PriorityQueue", "PriorityQueue") +@deprecation.deprecated_endpoints("PriorityQueue") class PriorityQueue(QueueBase): """A queue implementation that dequeues elements in prioritized order. @@ -1300,7 +1305,9 @@ class ConditionalAccumulator(ConditionalAccumulatorBase): return out -@tf_export("SparseConditionalAccumulator") +@tf_export("sparse.SparseConditionalAccumulator", + "SparseConditionalAccumulator") +@deprecation.deprecated_endpoints("SparseConditionalAccumulator") class SparseConditionalAccumulator(ConditionalAccumulatorBase): """A conditional accumulator for aggregating sparse gradients. diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index fff3d9b930..65bb77b474 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -43,6 +43,7 @@ from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import linalg_ops_impl from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops +from tensorflow.python.util import deprecation from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.deprecation import deprecated_arg_values from tensorflow.python.util.tf_export import tf_export @@ -341,6 +342,7 @@ class TruncatedNormal(Initializer): @tf_export("initializers.uniform_unit_scaling", "uniform_unit_scaling_initializer") +@deprecation.deprecated_endpoints("uniform_unit_scaling_initializer") class UniformUnitScaling(Initializer): """Initializer that generates tensors without scaling variance. @@ -401,6 +403,7 @@ class UniformUnitScaling(Initializer): @tf_export("keras.initializers.VarianceScaling", "initializers.variance_scaling", "variance_scaling_initializer") +@deprecation.deprecated_endpoints("variance_scaling_initializer") class VarianceScaling(Initializer): """Initializer capable of adapting its scale to the shape of weights tensors. @@ -494,6 +497,7 @@ class VarianceScaling(Initializer): @tf_export("keras.initializers.Orthogonal", "initializers.orthogonal", "orthogonal_initializer", "keras.initializers.orthogonal") +@deprecation.deprecated_endpoints("orthogonal_initializer") class Orthogonal(Initializer): """Initializer that generates an orthogonal matrix. @@ -1149,6 +1153,7 @@ class GlorotUniform(VarianceScaling): @tf_export("glorot_normal_initializer", "keras.initializers.glorot_normal", "initializers.glorot_normal") +@deprecation.deprecated_endpoints("glorot_normal_initializer") class GlorotNormal(VarianceScaling): """The Glorot normal initializer, also called Xavier normal initializer. diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index f4a93560be..bf4354fa73 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -80,6 +80,7 @@ def _RegularizedGramianCholesky(matrix, l2_regularizer, first_kind): @tf_export('cholesky_solve', 'linalg.cholesky_solve') +@deprecation.deprecated_endpoints('cholesky_solve') def cholesky_solve(chol, rhs, name=None): """Solves systems of linear eqns `A X = RHS`, given Cholesky factorizations. @@ -167,7 +168,8 @@ def eye(num_rows, name=name) -@tf_export('matrix_solve_ls', 'linalg.lstsq') +@tf_export('linalg.lstsq', 'matrix_solve_ls') +@deprecation.deprecated_endpoints('matrix_solve_ls') def matrix_solve_ls(matrix, rhs, l2_regularizer=0.0, fast=True, name=None): r"""Solves one or more linear least-squares problems. @@ -220,7 +222,7 @@ def matrix_solve_ls(matrix, rhs, l2_regularizer=0.0, fast=True, name=None): squares sense. Raises: - NotImplementedError: matrix_solve_ls is currently disabled for complex128 + NotImplementedError: linalg.lstsq is currently disabled for complex128 and l2_regularizer != 0 due to poor accuracy. """ @@ -303,7 +305,8 @@ def matrix_solve_ls(matrix, rhs, l2_regularizer=0.0, fast=True, name=None): matrix, rhs, l2_regularizer, fast=fast, name=name) -@tf_export('self_adjoint_eig', 'linalg.eigh') +@tf_export('linalg.eigh', 'self_adjoint_eig') +@deprecation.deprecated_endpoints('self_adjoint_eig') def self_adjoint_eig(tensor, name=None): """Computes the eigen decomposition of a batch of self-adjoint matrices. @@ -325,12 +328,13 @@ def self_adjoint_eig(tensor, name=None): return e, v -@tf_export('self_adjoint_eigvals', 'linalg.eigvalsh') +@tf_export('linalg.eigvalsh', 'self_adjoint_eigvals') +@deprecation.deprecated_endpoints('self_adjoint_eigvals') def self_adjoint_eigvals(tensor, name=None): """Computes the eigenvalues of one or more self-adjoint matrices. Note: If your program backpropagates through this function, you should replace - it with a call to tf.self_adjoint_eig (possibly ignoring the second output) to + it with a call to tf.linalg.eigvalsh (possibly ignoring the second output) to avoid computing the eigen decomposition twice. This is because the eigenvectors are used to compute the gradient w.r.t. the eigenvalues. See _SelfAdjointEigV2Grad in linalg_grad.py. @@ -348,6 +352,7 @@ def self_adjoint_eigvals(tensor, name=None): @tf_export('svd', 'linalg.svd') +@deprecation.deprecated_endpoints('svd') def svd(tensor, full_matrices=False, compute_uv=True, name=None): r"""Computes the singular value decompositions of one or more matrices. diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index 5443699ddd..cffaa983d4 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -59,7 +59,7 @@ def initialize_all_tables(name="init_all_tables"): return tables_initializer(name) -@tf_export("tables_initializer") +@tf_export("initializers.tables_initializer", "tables_initializer") def tables_initializer(name="init_all_tables"): """Returns an Op that initializes all tables of the default graph. diff --git a/tensorflow/python/ops/manip_ops.py b/tensorflow/python/ops/manip_ops.py index 6633565a64..d9d0728287 100644 --- a/tensorflow/python/ops/manip_ops.py +++ b/tensorflow/python/ops/manip_ops.py @@ -19,11 +19,13 @@ from __future__ import division from __future__ import print_function from tensorflow.python.ops import gen_manip_ops as _gen_manip_ops +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export # pylint: disable=protected-access -@tf_export('manip.roll') +@tf_export('roll', 'manip.roll') +@deprecation.deprecated_endpoints('manip.roll') def roll(input, shift, axis): # pylint: disable=redefined-builtin return _gen_manip_ops.roll(input, shift, axis) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index f57abf6704..83b8b5a3a4 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -70,7 +70,7 @@ def _set_doc(doc): # pylint: disable=redefined-builtin -@tf_export("argmax") +@tf_export("math.argmax", "argmax") @deprecation.deprecated_args(None, "Use the `axis` argument instead", "dimension") @_set_doc( @@ -88,7 +88,7 @@ def argmax(input, return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) -@tf_export("argmin") +@tf_export("math.argmin", "argmin") @deprecation.deprecated_args(None, "Use the `axis` argument instead", "dimension") @_set_doc( @@ -111,7 +111,7 @@ def argmin(input, # pylint: disable=anomalous-backslash-in-string,protected-access # pylint: disable=g-docstring-has-escape -@tf_export("abs") +@tf_export("math.abs", "abs") def abs(x, name=None): # pylint: disable=redefined-builtin r"""Computes the absolute value of a tensor. @@ -186,7 +186,7 @@ class DivideDelegateWithName(object): return _div_python2(self.x, y, self.name) -@tf_export("divide") +@tf_export("math.divide", "divide") def divide(x, y, name=None): """Computes Python style division of `x` by `y`.""" @@ -198,7 +198,7 @@ def divide(x, y, name=None): return x / y -@tf_export("multiply") +@tf_export("math.multiply", "multiply") def multiply(x, y, name=None): return gen_math_ops.mul(x, y, name) @@ -218,7 +218,7 @@ _mul.__doc__ = ( gen_math_ops.mul.__doc__ + ("" if _mul.__doc__ is None else _mul.__doc__)) -@tf_export("subtract") +@tf_export("math.subtract", "subtract") def subtract(x, y, name=None): return gen_math_ops.sub(x, y, name) @@ -239,7 +239,7 @@ _sub.__doc__ = ( # pylint: disable=g-docstring-has-escape -@tf_export("negative") +@tf_export("math.negative", "negative") def negative(x, name=None): """Computes numerical negative value element-wise. @@ -288,7 +288,7 @@ def _neg(x, name=None): # pylint: enable=g-docstring-has-escape -@tf_export("sign") +@tf_export("math.sign", "sign") def sign(x, name=None): """Returns an element-wise indication of the sign of a number. @@ -319,7 +319,7 @@ def sign(x, name=None): return gen_math_ops.sign(x, name=name) -@tf_export("square") +@tf_export("math.square", "square") def square(x, name=None): r"""Computes square of x element-wise. @@ -342,7 +342,7 @@ def square(x, name=None): return gen_math_ops.square(x, name=name) -@tf_export("sqrt") +@tf_export("math.sqrt", "sqrt") def sqrt(x, name=None): r"""Computes square root of x element-wise. @@ -365,7 +365,8 @@ def sqrt(x, name=None): return gen_math_ops.sqrt(x, name=name) -@tf_export("erf") +@tf_export("math.erf", "erf") +@deprecation.deprecated_endpoints("erf") def erf(x, name=None): """Computes the Gauss error function of `x` element-wise. @@ -386,7 +387,7 @@ def erf(x, name=None): return gen_math_ops.erf(x, name=name) -@tf_export("scalar_mul") +@tf_export("math.scalar_mul", "scalar_mul") def scalar_mul(scalar, x): """Multiplies a scalar times a `Tensor` or `IndexedSlices` object. @@ -416,7 +417,7 @@ def scalar_mul(scalar, x): raise ValueError("Only scalar multiply works, got shape %s" % shape) -@tf_export("pow") +@tf_export("math.pow", "pow") def pow(x, y, name=None): # pylint: disable=redefined-builtin r"""Computes the power of one value to another. @@ -444,7 +445,7 @@ def pow(x, y, name=None): # pylint: disable=redefined-builtin # pylint: disable=redefined-builtin,redefined-outer-name -@tf_export("complex") +@tf_export("dtypes.complex", "complex") def complex(real, imag, name=None): r"""Converts two real numbers to a complex number. @@ -486,7 +487,8 @@ def complex(real, imag, name=None): return gen_math_ops._complex(real, imag, Tout=Tout, name=name) -@tf_export("real") +@tf_export("math.real", "real") +@deprecation.deprecated_endpoints("real") def real(input, name=None): r"""Returns the real part of a complex (or real) tensor. @@ -517,7 +519,8 @@ def real(input, name=None): return input -@tf_export("imag") +@tf_export("math.imag", "imag") +@deprecation.deprecated_endpoints("imag") def imag(input, name=None): r"""Returns the imaginary part of a complex (or real) tensor. @@ -547,7 +550,8 @@ def imag(input, name=None): return array_ops.zeros_like(input) -@tf_export("angle") +@tf_export("math.angle", "angle") +@deprecation.deprecated_endpoints("angle") def angle(input, name=None): r"""Returns the element-wise argument of a complex (or real) tensor. @@ -586,7 +590,7 @@ def angle(input, name=None): # pylint: enable=redefined-outer-name,redefined-builtin -@tf_export("round") +@tf_export("math.round", "round") def round(x, name=None): # pylint: disable=redefined-builtin """Rounds the values of a tensor to the nearest integer, element-wise. @@ -613,7 +617,7 @@ def round(x, name=None): # pylint: disable=redefined-builtin return gen_math_ops.round(x, name=name) -@tf_export("cast") +@tf_export("dtypes.cast", "cast") def cast(x, dtype, name=None): """Casts a tensor to a new type. @@ -676,7 +680,7 @@ def cast(x, dtype, name=None): return x -@tf_export("saturate_cast") +@tf_export("dtypes.saturate_cast", "saturate_cast") def saturate_cast(value, dtype, name=None): """Performs a safe saturating cast of `value` to `dtype`. @@ -995,7 +999,7 @@ def _div_python2(x, y, name=None): return gen_math_ops.floor_div(x, y, name=name) -@tf_export("truediv") +@tf_export("math.truediv", "truediv") def truediv(x, y, name=None): """Divides x / y elementwise (using Python 3 division operator semantics). @@ -1006,7 +1010,7 @@ def truediv(x, y, name=None): arguments are cast to floating types first. This op is generated by normal `x / y` division in Python 3 and in Python 2.7 with `from __future__ import division`. If you want integer division that rounds - down, use `x // y` or `tf.floordiv`. + down, use `x // y` or `tf.math.floordiv`. `x` and `y` must have the same numeric type. If the inputs are floating point, the output will have the same type. If the inputs are integral, the @@ -1078,7 +1082,8 @@ mod = gen_math_ops.floor_mod # TODO(aselle): Deprecate this once all internal functionality uses # tf.truncatediv -@tf_export("floordiv") +@tf_export("math.floordiv", "floordiv") +@deprecation.deprecated_endpoints("floordiv") def floordiv(x, y, name=None): """Divides `x / y` elementwise, rounding toward the most negative integer. @@ -1151,7 +1156,8 @@ _OverrideBinaryOperatorHelper(gen_math_ops.floor_mod, "mod") _OverrideBinaryOperatorHelper(pow, "pow") -@tf_export("logical_xor") +@tf_export("math.logical_xor", "logical_xor") +@deprecation.deprecated_endpoints("logical_xor") def logical_xor(x, y, name="LogicalXor"): """x ^ y = (x | y) & ~(x & y).""" # TODO(alemi) Make this a cwise op if people end up relying on it. @@ -1277,7 +1283,7 @@ def _may_reduce_to_scalar(keepdims, axis, reduction_indices, output): return output -@tf_export("reduce_sum") +@tf_export("math.reduce_sum", "reduce_sum") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_sum(input_tensor, @@ -1339,7 +1345,7 @@ def reduce_sum(input_tensor, name=name)) -@tf_export("count_nonzero") +@tf_export("math.count_nonzero", "count_nonzero") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def count_nonzero(input_tensor, @@ -1417,7 +1423,7 @@ def count_nonzero(input_tensor, dtype=dtype) -@tf_export("reduce_mean") +@tf_export("math.reduce_mean", "reduce_mean") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_mean(input_tensor, @@ -1489,7 +1495,7 @@ def reduce_mean(input_tensor, name=name)) -@tf_export("reduce_prod") +@tf_export("math.reduce_prod", "reduce_prod") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_prod(input_tensor, @@ -1539,7 +1545,7 @@ def reduce_prod(input_tensor, name=name)) -@tf_export("reduce_min") +@tf_export("math.reduce_min", "reduce_min") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_min(input_tensor, @@ -1588,7 +1594,7 @@ def reduce_min(input_tensor, name=name)) -@tf_export("reduce_max") +@tf_export("math.reduce_max", "reduce_max") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_max(input_tensor, @@ -1637,7 +1643,7 @@ def reduce_max(input_tensor, name=name)) -@tf_export("reduce_all") +@tf_export("math.reduce_all", "reduce_all") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_all(input_tensor, @@ -1695,7 +1701,7 @@ def reduce_all(input_tensor, name=name)) -@tf_export("reduce_any") +@tf_export("math.reduce_any", "reduce_any") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_any(input_tensor, @@ -1753,7 +1759,7 @@ def reduce_any(input_tensor, name=name)) -@tf_export("reduce_logsumexp") +@tf_export("math.reduce_logsumexp", "reduce_logsumexp") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def reduce_logsumexp(input_tensor, @@ -1827,7 +1833,8 @@ def reduce_logsumexp(input_tensor, return _may_reduce_to_scalar(keepdims, axis, reduction_indices, result) -@tf_export("trace", "linalg.trace") +@tf_export("linalg.trace", "trace") +@deprecation.deprecated_endpoints("trace") def trace(x, name=None): """Compute the trace of a tensor `x`. @@ -1841,12 +1848,12 @@ def trace(x, name=None): ```python x = tf.constant([[1, 2], [3, 4]]) - tf.trace(x) # 5 + tf.linalg.trace(x) # 5 x = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) - tf.trace(x) # 15 + tf.linalg.trace(x) # 15 x = tf.constant([[[1, 2, 3], [4, 5, 6], @@ -1854,7 +1861,7 @@ def trace(x, name=None): [[-1, -2, -3], [-4, -5, -6], [-7, -8, -9]]]) - tf.trace(x) # [15, -15] + tf.linalg.trace(x) # [15, -15] ``` Args: @@ -1869,7 +1876,7 @@ def trace(x, name=None): return reduce_sum(array_ops.matrix_diag_part(x), [-1], name=name) -@tf_export("matmul") +@tf_export("linalg.matmul", "matmul") def matmul(a, b, transpose_a=False, @@ -2131,7 +2138,7 @@ def _as_indexed_slices_list(inputs, optimize=True): return casted_outputs -@tf_export("add_n") +@tf_export("math.add_n", "add_n") def add_n(inputs, name=None): """Adds all input tensors element-wise. @@ -2166,14 +2173,15 @@ def add_n(inputs, name=None): return gen_math_ops.add_n(inputs, name=name) -@tf_export("accumulate_n") +@tf_export("math.accumulate_n", "accumulate_n") +@deprecation.deprecated_endpoints("accumulate_n") def accumulate_n(inputs, shape=None, tensor_dtype=None, name=None): """Returns the element-wise sum of a list of tensors. Optionally, pass `shape` and `tensor_dtype` for shape and type checking, otherwise, these are inferred. - `tf.accumulate_n` performs the same operation as `tf.add_n`, but does not + `tf.math.accumulate_n` performs the same operation as `tf.add_n`, but does not wait for all of its inputs to be ready before beginning to sum. This can save memory if inputs are ready at different times, since minimum temporary storage is proportional to the output size rather than the inputs size. @@ -2185,10 +2193,10 @@ def accumulate_n(inputs, shape=None, tensor_dtype=None, name=None): ```python a = tf.constant([[1, 2], [3, 4]]) b = tf.constant([[5, 0], [0, 6]]) - tf.accumulate_n([a, b, a]) # [[7, 4], [6, 14]] + tf.math.accumulate_n([a, b, a]) # [[7, 4], [6, 14]] # Explicitly pass shape and type - tf.accumulate_n([a, b, a], shape=[2, 2], tensor_dtype=tf.int32) + tf.math.accumulate_n([a, b, a], shape=[2, 2], tensor_dtype=tf.int32) # [[7, 4], # [6, 14]] ``` @@ -2252,7 +2260,7 @@ def _accumulate_n_grad(op, grad): return [grad] * len(op.inputs) -@tf_export("nn.sigmoid", "sigmoid") +@tf_export("math.sigmoid", "nn.sigmoid", "sigmoid") def sigmoid(x, name=None): """Computes sigmoid of `x` element-wise. @@ -2275,7 +2283,8 @@ def sigmoid(x, name=None): return gen_math_ops.sigmoid(x, name=name) -@tf_export("log_sigmoid") +@tf_export("math.log_sigmoid", "log_sigmoid") +@deprecation.deprecated_endpoints("log_sigmoid") def log_sigmoid(x, name=None): """Computes log sigmoid of `x` element-wise. @@ -2294,7 +2303,7 @@ def log_sigmoid(x, name=None): return gen_math_ops.neg(gen_nn_ops.softplus(-x), name=name) -@tf_export("nn.tanh", "tanh") +@tf_export("math.tanh", "nn.tanh", "tanh") def tanh(x, name=None): """Computes hyperbolic tangent of `x` element-wise. @@ -2315,7 +2324,8 @@ def tanh(x, name=None): return gen_math_ops.tanh(x, name=name) -@tf_export("bincount") +@tf_export("math.bincount", "bincount") +@deprecation.deprecated_endpoints("bincount") def bincount(arr, weights=None, minlength=None, @@ -2362,7 +2372,7 @@ def bincount(arr, return gen_math_ops.bincount(arr, output_size, weights) -@tf_export("cumsum") +@tf_export("math.cumsum", "cumsum") def cumsum(x, axis=0, exclusive=False, reverse=False, name=None): """Compute the cumulative sum of the tensor `x` along `axis`. @@ -2414,7 +2424,8 @@ def cumsum(x, axis=0, exclusive=False, reverse=False, name=None): x, axis, exclusive=exclusive, reverse=reverse, name=name) -@tf_export("cumprod") +@tf_export("math.cumprod", "cumprod") +@deprecation.deprecated_endpoints("cumprod") def cumprod(x, axis=0, exclusive=False, reverse=False, name=None): """Compute the cumulative product of the tensor `x` along `axis`. @@ -2422,7 +2433,7 @@ def cumprod(x, axis=0, exclusive=False, reverse=False, name=None): first element of the input is identical to the first element of the output: ```python - tf.cumprod([a, b, c]) # [a, a * b, a * b * c] + tf.math.cumprod([a, b, c]) # [a, a * b, a * b * c] ``` By setting the `exclusive` kwarg to `True`, an exclusive cumprod is @@ -2430,21 +2441,21 @@ def cumprod(x, axis=0, exclusive=False, reverse=False, name=None): instead: ```python - tf.cumprod([a, b, c], exclusive=True) # [1, a, a * b] + tf.math.cumprod([a, b, c], exclusive=True) # [1, a, a * b] ``` By setting the `reverse` kwarg to `True`, the cumprod is performed in the opposite direction: ```python - tf.cumprod([a, b, c], reverse=True) # [a * b * c, b * c, c] + tf.math.cumprod([a, b, c], reverse=True) # [a * b * c, b * c, c] ``` This is more efficient than using separate `tf.reverse` ops. The `reverse` and `exclusive` kwargs can also be combined: ```python - tf.cumprod([a, b, c], exclusive=True, reverse=True) # [b * c, c, 1] + tf.math.cumprod([a, b, c], exclusive=True, reverse=True) # [b * c, c, 1] ``` Args: @@ -2466,7 +2477,8 @@ def cumprod(x, axis=0, exclusive=False, reverse=False, name=None): x, axis, exclusive=exclusive, reverse=reverse, name=name) -@tf_export("conj") +@tf_export("math.conj", "conj") +@deprecation.deprecated_endpoints("conj") def conj(x, name=None): r"""Returns the complex conjugate of a complex number. @@ -2480,7 +2492,7 @@ def conj(x, name=None): For example: # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j] - tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j] + tf.math.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j] If `x` is real, it is returned unchanged. @@ -2566,7 +2578,8 @@ def _unsorted_segment_N(data, segment_ids, num_segments): return gen_math_ops.maximum(N, 1) -@tf_export("unsorted_segment_mean") +@tf_export("math.unsorted_segment_mean", "unsorted_segment_mean") +@deprecation.deprecated_endpoints("unsorted_segment_mean") def unsorted_segment_mean(data, segment_ids, num_segments, name=None): r"""Computes the mean along segments of a tensor. @@ -2608,7 +2621,8 @@ def unsorted_segment_mean(data, segment_ids, num_segments, name=None): return summed / N -@tf_export("unsorted_segment_sqrt_n") +@tf_export("math.unsorted_segment_sqrt_n", "unsorted_segment_sqrt_n") +@deprecation.deprecated_endpoints("unsorted_segment_sqrt_n") def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None): r"""Computes the sum along segments of a tensor divided by the sqrt(N). @@ -2653,7 +2667,8 @@ def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None): return summed / gen_math_ops.sqrt(N) -@tf_export("sparse_segment_sum") +@tf_export("sparse.segment_sum", "sparse_segment_sum") +@deprecation.deprecated_endpoints("sparse_segment_sum") def sparse_segment_sum(data, indices, segment_ids, name=None, num_segments=None): r"""Computes the sum along sparse segments of a tensor. @@ -2674,16 +2689,16 @@ def sparse_segment_sum(data, indices, segment_ids, name=None, c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]]) # Select two rows, one segment. - tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0])) + tf.sparse.segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0])) # => [[0 0 0 0]] # Select two rows, two segment. - tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 1])) + tf.sparse.segment_sum(c, tf.constant([0, 1]), tf.constant([0, 1])) # => [[ 1 2 3 4] # [-1 -2 -3 -4]] # With missing segment ids. - tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 2]), + tf.sparse.segment_sum(c, tf.constant([0, 1]), tf.constant([0, 2]), num_segments=4) # => [[ 1 2 3 4] # [ 0 0 0 0] @@ -2691,7 +2706,7 @@ def sparse_segment_sum(data, indices, segment_ids, name=None, # [ 0 0 0 0]] # Select all rows, two segments. - tf.sparse_segment_sum(c, tf.constant([0, 1, 2]), tf.constant([0, 0, 1])) + tf.sparse.segment_sum(c, tf.constant([0, 1, 2]), tf.constant([0, 0, 1])) # => [[0 0 0 0] # [5 6 7 8]] @@ -2726,7 +2741,8 @@ def sparse_segment_sum(data, indices, segment_ids, name=None, data=data, indices=indices, segment_ids=segment_ids, name=name) -@tf_export("sparse_segment_mean") +@tf_export("sparse.segment_mean", "sparse_segment_mean") +@deprecation.deprecated_endpoints("sparse_segment_mean") def sparse_segment_mean(data, indices, segment_ids, @@ -2771,7 +2787,8 @@ def sparse_segment_mean(data, data=data, indices=indices, segment_ids=segment_ids, name=name) -@tf_export("sparse_segment_sqrt_n") +@tf_export("sparse.segment_sqrt_n", "sparse_segment_sqrt_n") +@deprecation.deprecated_endpoints("sparse_segment_sqrt_n") def sparse_segment_sqrt_n(data, indices, segment_ids, diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index 2a1919e66f..453848fc00 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -328,7 +328,7 @@ def swish(features): return features * math_ops.sigmoid(features) -@tf_export("nn.l2_normalize") +@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize") @deprecated_args(None, "dim is deprecated, use axis instead", "dim") def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None): """Normalizes along dimension `axis` using an L2 norm. @@ -360,7 +360,7 @@ def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None): return math_ops.multiply(x, x_inv_norm, name=name) -@tf_export("nn.zero_fraction") +@tf_export("math.zero_fraction", "nn.zero_fraction") def zero_fraction(value, name=None): """Returns the fraction of zeros in `value`. @@ -689,7 +689,7 @@ def moments( # Compute true mean while keeping the dims for proper broadcasting. mean = math_ops.reduce_mean(y, axes, keepdims=True, name="mean") # sample variance, not unbiased variance - # Note: stop_gradient does not change the gradient that gets + # Note: stop_gradient does not change the gradient that gets # backpropagated to the mean from the variance calculation, # because that gradient is zero variance = math_ops.reduce_mean( diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 9ef177e97b..fd71e7cc39 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1692,7 +1692,7 @@ def _softmax(logits, compute_op, dim=-1, name=None): return output -@tf_export("nn.softmax") +@tf_export("nn.softmax", "math.softmax") @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim") def softmax(logits, axis=None, name=None, dim=None): """Computes softmax activations. @@ -1722,7 +1722,7 @@ def softmax(logits, axis=None, name=None, dim=None): return _softmax(logits, gen_nn_ops.softmax, axis, name) -@tf_export("nn.log_softmax") +@tf_export("nn.log_softmax", "math.log_softmax") @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim") def log_softmax(logits, axis=None, name=None, dim=None): """Computes log softmax activations. @@ -2329,7 +2329,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di return ret -@tf_export("nn.top_k") +@tf_export("math.top_k", "nn.top_k") def top_k(input, k=1, sorted=True, name=None): # pylint: disable=redefined-builtin """Finds values and indices of the `k` largest entries for the last dimension. @@ -2644,7 +2644,7 @@ def erosion2d(value, kernel, strides, rates, padding, name=None): name=name)) -@tf_export("nn.in_top_k") +@tf_export("math.in_top_k", "nn.in_top_k") def in_top_k(predictions, targets, k, name=None): r"""Says whether the targets are in the top `K` predictions. diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py index 8fcbd7d834..002e87b411 100644 --- a/tensorflow/python/ops/numerics.py +++ b/tensorflow/python/ops/numerics.py @@ -24,10 +24,12 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export("verify_tensor_all_finite") +@tf_export("debugging.assert_all_finite", "verify_tensor_all_finite") +@deprecation.deprecated_endpoints("verify_tensor_all_finite") def verify_tensor_all_finite(t, msg, name=None): """Assert that the tensor does not contain any NaN's or Inf's. diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index b3e03a0135..ff50fe0d09 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -36,6 +36,7 @@ from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.gen_parsing_ops import * # pylint: enable=wildcard-import,undefined-variable from tensorflow.python.platform import tf_logging +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -45,7 +46,7 @@ ops.NotDifferentiable("SerializeTensor") ops.NotDifferentiable("StringToNumber") -@tf_export("VarLenFeature") +@tf_export("io.VarLenFeature", "VarLenFeature") class VarLenFeature(collections.namedtuple("VarLenFeature", ["dtype"])): """Configuration for parsing a variable-length input feature. @@ -55,7 +56,7 @@ class VarLenFeature(collections.namedtuple("VarLenFeature", ["dtype"])): pass -@tf_export("SparseFeature") +@tf_export("io.SparseFeature", "SparseFeature") class SparseFeature( collections.namedtuple( "SparseFeature", @@ -130,7 +131,7 @@ class SparseFeature( cls, index_key, value_key, dtype, size, already_sorted) -@tf_export("FixedLenFeature") +@tf_export("io.FixedLenFeature", "FixedLenFeature") class FixedLenFeature(collections.namedtuple( "FixedLenFeature", ["shape", "dtype", "default_value"])): """Configuration for parsing a fixed-length input feature. @@ -150,7 +151,7 @@ class FixedLenFeature(collections.namedtuple( cls, shape, dtype, default_value) -@tf_export("FixedLenSequenceFeature") +@tf_export("io.FixedLenSequenceFeature", "FixedLenSequenceFeature") class FixedLenSequenceFeature(collections.namedtuple( "FixedLenSequenceFeature", ["shape", "dtype", "allow_missing", "default_value"])): @@ -360,7 +361,7 @@ def _prepend_none_dimension(features): return features -@tf_export("parse_example") +@tf_export("io.parse_example", "parse_example") def parse_example(serialized, features, name=None, example_names=None): # pylint: disable=line-too-long """Parses `Example` protos into a `dict` of tensors. @@ -761,7 +762,7 @@ def _process_raw_parameters(names, dense_defaults, sparse_keys, sparse_types, dense_shapes_as_proto, dense_shapes) -@tf_export("parse_single_example") +@tf_export("io.parse_single_example", "parse_single_example") def parse_single_example(serialized, features, name=None, example_names=None): """Parses a single `Example` proto. @@ -1244,7 +1245,7 @@ def _parse_sequence_example_raw(serialized, # TODO(sundberg): rewrite this method to call the batch version, which is more # efficient especially for large inputs. -@tf_export("parse_single_sequence_example") +@tf_export("io.parse_single_sequence_example", "parse_single_sequence_example") def parse_single_sequence_example( serialized, context_features=None, sequence_features=None, example_name=None, name=None): @@ -1564,7 +1565,8 @@ def _parse_single_sequence_example_raw(serialized, # Swap `name` and `na_value` for backward compatibility. -@tf_export("decode_csv") +@tf_export("io.decode_csv", "decode_csv") +@deprecation.deprecated_endpoints("decode_csv") def decode_csv(records, record_defaults, field_delim=",", diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py index 4baf506385..c2eb9dfc5d 100644 --- a/tensorflow/python/ops/random_ops.py +++ b/tensorflow/python/ops/random_ops.py @@ -29,6 +29,7 @@ from tensorflow.python.ops import math_ops # go/tf-wildcard-import # pylint: disable=wildcard-import from tensorflow.python.ops.gen_random_ops import * +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export # pylint: enable=wildcard-import @@ -43,7 +44,7 @@ def _ShapeTensor(shape): return ops.convert_to_tensor(shape, dtype=dtype, name="shape") -@tf_export("random_normal") +@tf_export("random.normal", "random_normal") def random_normal(shape, mean=0.0, stddev=1.0, @@ -136,7 +137,7 @@ def parameterized_truncated_normal(shape, return rnd -@tf_export("truncated_normal") +@tf_export("random.truncated_normal", "truncated_normal") def truncated_normal(shape, mean=0.0, stddev=1.0, @@ -181,7 +182,7 @@ ops.NotDifferentiable("ParameterizedTruncatedNormal") ops.NotDifferentiable("TruncatedNormal") -@tf_export("random_uniform") +@tf_export("random.uniform", "random_uniform") def random_uniform(shape, minval=0, maxval=None, @@ -246,7 +247,7 @@ def random_uniform(shape, ops.NotDifferentiable("RandomUniform") -@tf_export("random_shuffle") +@tf_export("random.shuffle", "random_shuffle") def random_shuffle(value, seed=None, name=None): """Randomly shuffles a tensor along its first dimension. @@ -277,7 +278,7 @@ def random_shuffle(value, seed=None, name=None): value, seed=seed1, seed2=seed2, name=name) -@tf_export("random_crop") +@tf_export("image.random_crop", "random_crop") def random_crop(value, size, seed=None, name=None): """Randomly crops a tensor to a given size. @@ -320,7 +321,7 @@ def random_crop(value, size, seed=None, name=None): return array_ops.slice(value, offset, size, name=name) -@tf_export("multinomial") +@tf_export("random.multinomial", "multinomial") def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): """Draws samples from a multinomial distribution. @@ -356,7 +357,8 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): ops.NotDifferentiable("Multinomial") -@tf_export("random_gamma") +@tf_export("random.gamma", "random_gamma") +@deprecation.deprecated_endpoints("random_gamma") def random_gamma(shape, alpha, beta=None, @@ -439,7 +441,8 @@ def random_gamma(shape, shape, alpha_broadcast, seed=seed1, seed2=seed2) / beta) -@tf_export("random_poisson") +@tf_export("random.poisson", "random_poisson") +@deprecation.deprecated_endpoints("random_poisson") def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None): """Draws `shape` samples from each of the given Poisson distribution(s). diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index 400a42a3c0..7e3dbdbad4 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -185,7 +185,8 @@ def sparse_eye(num_rows, # pylint: disable=protected-access -@tf_export("sparse_concat") +@tf_export("sparse.concat", "sparse_concat") +@deprecation.deprecated_endpoints("sparse_concat") @deprecation.deprecated_args( None, "concat_dim is deprecated, use axis instead", "concat_dim") def sparse_concat(axis, @@ -317,7 +318,8 @@ def sparse_concat(axis, return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) -@tf_export("sparse_add") +@tf_export("sparse.add", "sparse_add") +@deprecation.deprecated_endpoints("sparse_add") def sparse_add(a, b, thresh=0): """Adds two tensors, at least one of each is a `SparseTensor`. @@ -557,7 +559,8 @@ def sparse_dense_cwise_add(sp_t, dense_t): return sparse_tensor.SparseTensor(sp_t.indices, result, sp_t.dense_shape) -@tf_export("sparse_reorder") +@tf_export("sparse.reorder", "sparse_reorder") +@deprecation.deprecated_endpoints("sparse_reorder") def sparse_reorder(sp_input, name=None): """Reorders a `SparseTensor` into the canonical, row-major ordering. @@ -607,7 +610,8 @@ def sparse_reorder(sp_input, name=None): return sparse_tensor.SparseTensor(reordered_ind, reordered_val, dense_shape) -@tf_export("sparse_reshape") +@tf_export("sparse.reshape", "sparse_reshape") +@deprecation.deprecated_endpoints("sparse_reshape") def sparse_reshape(sp_input, shape, name=None): """Reshapes a `SparseTensor` to represent values in a new dense shape. @@ -700,7 +704,8 @@ class KeywordRequired(object): return "KeywordRequired()" -@tf_export("sparse_split") +@tf_export("sparse.split", "sparse_split") +@deprecation.deprecated_endpoints("sparse_split") @deprecation.deprecated_args( None, "split_dim is deprecated, use axis instead", "split_dim") def sparse_split(keyword_required=KeywordRequired(), @@ -773,7 +778,8 @@ def sparse_split(keyword_required=KeywordRequired(), return sparse_tensors -@tf_export("sparse_slice") +@tf_export("sparse.slice", "sparse_slice") +@deprecation.deprecated_endpoints("sparse_slice") def sparse_slice(sp_input, start, size, name=None): """Slice a `SparseTensor` based on the `start` and `size. @@ -785,11 +791,11 @@ def sparse_slice(sp_input, start, size, name=None): Graphically the output tensors are: - sparse_slice([0, 0], [2, 4]) = shape = [2, 4] + sparse.slice([0, 0], [2, 4]) = shape = [2, 4] [ a ] [b c ] - sparse_slice([0, 4], [2, 3]) = shape = [2, 3] + sparse.slice([0, 4], [2, 3]) = shape = [2, 3] [ d e ] [ ] @@ -823,6 +829,9 @@ def sparse_slice(sp_input, start, size, name=None): @tf_export("sparse_to_dense") +@deprecation.deprecated( + None, + "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.") def sparse_to_dense(sparse_indices, output_shape, sparse_values, @@ -878,7 +887,8 @@ def sparse_to_dense(sparse_indices, name=name) -@tf_export("sparse_reduce_max") +@tf_export("sparse.reduce_max", "sparse_reduce_max") +@deprecation.deprecated_endpoints("sparse_reduce_max") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def sparse_reduce_max(sp_input, axis=None, keepdims=None, @@ -912,16 +922,16 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None, # 'x' represents [[1, ?, 2] # [?, 3, ?]] # where ? is implicitly-zero. - tf.sparse_reduce_max(x) ==> 3 - tf.sparse_reduce_max(x, 0) ==> [1, 3, 2] - tf.sparse_reduce_max(x, 1) ==> [2, 3] # Can also use -1 as the axis. - tf.sparse_reduce_max(x, 1, keepdims=True) ==> [[2], [3]] - tf.sparse_reduce_max(x, [0, 1]) ==> 3 + tf.sparse.reduce_max(x) ==> 3 + tf.sparse.reduce_max(x, 0) ==> [1, 3, 2] + tf.sparse.reduce_max(x, 1) ==> [2, 3] # Can also use -1 as the axis. + tf.sparse.reduce_max(x, 1, keepdims=True) ==> [[2], [3]] + tf.sparse.reduce_max(x, [0, 1]) ==> 3 # 'y' represents [[-7, ?] # [ 4, 3] # [ ?, ?] - tf.sparse_reduce_max(x, 1) ==> [-7, 4, 0] + tf.sparse.reduce_max(x, 1) ==> [-7, 4, 0] ``` Args: @@ -945,7 +955,8 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None, math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims) -@tf_export("sparse_reduce_max_sparse") +@tf_export("sparse.reduce_max_sparse", "sparse_reduce_max_sparse") +@deprecation.deprecated_endpoints("sparse_reduce_max_sparse") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def sparse_reduce_max_sparse(sp_input, @@ -995,7 +1006,8 @@ def sparse_reduce_max_sparse(sp_input, return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) -@tf_export("sparse_reduce_sum") +@tf_export("sparse.reduce_sum", "sparse_reduce_sum") +@deprecation.deprecated_endpoints("sparse_reduce_sum") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def sparse_reduce_sum(sp_input, axis=None, keepdims=None, @@ -1021,11 +1033,11 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None, # 'x' represents [[1, ?, 1] # [?, 1, ?]] # where ? is implicitly-zero. - tf.sparse_reduce_sum(x) ==> 3 - tf.sparse_reduce_sum(x, 0) ==> [1, 1, 1] - tf.sparse_reduce_sum(x, 1) ==> [2, 1] # Can also use -1 as the axis. - tf.sparse_reduce_sum(x, 1, keepdims=True) ==> [[2], [1]] - tf.sparse_reduce_sum(x, [0, 1]) ==> 3 + tf.sparse.reduce_sum(x) ==> 3 + tf.sparse.reduce_sum(x, 0) ==> [1, 1, 1] + tf.sparse.reduce_sum(x, 1) ==> [2, 1] # Can also use -1 as the axis. + tf.sparse.reduce_sum(x, 1, keepdims=True) ==> [[2], [1]] + tf.sparse.reduce_sum(x, [0, 1]) ==> 3 ``` Args: @@ -1049,7 +1061,8 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None, math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims) -@tf_export("sparse_reduce_sum_sparse") +@tf_export("sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse") +@deprecation.deprecated_endpoints("sparse_reduce_sum_sparse") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def sparse_reduce_sum_sparse(sp_input, @@ -1099,7 +1112,8 @@ def sparse_reduce_sum_sparse(sp_input, return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) -@tf_export("sparse_tensor_to_dense") +@tf_export("sparse.to_dense", "sparse_tensor_to_dense") +@deprecation.deprecated_endpoints("sparse_tensor_to_dense") def sparse_tensor_to_dense(sp_input, default_value=0, validate_indices=True, @@ -1151,7 +1165,8 @@ def sparse_tensor_to_dense(sp_input, name=name) -@tf_export("sparse_to_indicator") +@tf_export("sparse.to_indicator", "sparse_to_indicator") +@deprecation.deprecated_endpoints("sparse_to_indicator") def sparse_to_indicator(sp_input, vocab_size, name=None): """Converts a `SparseTensor` of ids into a dense bool indicator tensor. @@ -1214,7 +1229,8 @@ def sparse_to_indicator(sp_input, vocab_size, name=None): sp_new, default_value=False, validate_indices=False, name=name) -@tf_export("sparse_merge") +@tf_export("sparse.merge", "sparse_merge") +@deprecation.deprecated_endpoints("sparse_merge") def sparse_merge(sp_ids, sp_values, vocab_size, name=None, already_sorted=False): """Combines a batch of feature ids and values into a single `SparseTensor`. @@ -1358,7 +1374,8 @@ def sparse_merge(sp_ids, sp_values, vocab_size, name=None, sorted_result.indices, sorted_result.values, new_shape) -@tf_export("sparse_retain") +@tf_export("sparse.retain", "sparse_retain") +@deprecation.deprecated_endpoints("sparse_retain") def sparse_retain(sp_input, to_retain): """Retains specified non-empty values within a `SparseTensor`. @@ -1402,7 +1419,8 @@ def sparse_retain(sp_input, to_retain): array_ops.identity(sp_input.dense_shape)) -@tf_export("sparse_reset_shape") +@tf_export("sparse.reset_shape", "sparse_reset_shape") +@deprecation.deprecated_endpoints("sparse_reset_shape") def sparse_reset_shape(sp_input, new_shape=None): """Resets the shape of a `SparseTensor` with indices and values unchanged. @@ -1503,7 +1521,8 @@ def sparse_reset_shape(sp_input, new_shape=None): return sparse_tensor.SparseTensor(in_indices, in_values, output_shape_tensor) -@tf_export("sparse_fill_empty_rows") +@tf_export("sparse.fill_empty_rows", "sparse_fill_empty_rows") +@deprecation.deprecated_endpoints("sparse_fill_empty_rows") def sparse_fill_empty_rows(sp_input, default_value, name=None): """Fills empty rows in the input 2-D `SparseTensor` with a default value. @@ -1567,7 +1586,8 @@ def sparse_fill_empty_rows(sp_input, default_value, name=None): dense_shape=sp_input.dense_shape), empty_row_indicator) -@tf_export("serialize_sparse") +@tf_export("io.serialize_sparse", "serialize_sparse") +@deprecation.deprecated_endpoints("serialize_sparse") def serialize_sparse(sp_input, name=None, out_type=dtypes.string): """Serialize a `SparseTensor` into a 3-vector (1-D `Tensor`) object. @@ -1593,7 +1613,8 @@ def serialize_sparse(sp_input, name=None, out_type=dtypes.string): out_type=out_type) -@tf_export("serialize_many_sparse") +@tf_export("io.serialize_many_sparse", "serialize_many_sparse") +@deprecation.deprecated_endpoints("serialize_many_sparse") def serialize_many_sparse(sp_input, name=None, out_type=dtypes.string): """Serialize `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor`. @@ -1694,7 +1715,8 @@ def deserialize_sparse(serialized_sparse, dtype, rank=None, name=None): return sparse_tensor.SparseTensor(output_indices, output_values, output_shape) -@tf_export("deserialize_many_sparse") +@tf_export("io.deserialize_many_sparse", "deserialize_many_sparse") +@deprecation.deprecated_endpoints("deserialize_many_sparse") def deserialize_many_sparse(serialized_sparse, dtype, rank=None, name=None): """Deserialize and concatenate `SparseTensors` from a serialized minibatch. @@ -1712,7 +1734,7 @@ def deserialize_many_sparse(serialized_sparse, dtype, rank=None, name=None): The input `SparseTensor` objects' indices are assumed ordered in standard lexicographic order. If this is not the case, after this - step run `sparse_reorder` to restore index ordering. + step run `sparse.reorder` to restore index ordering. For example, if the serialized input is a `[2, 3]` matrix representing two original `SparseTensor` objects: @@ -1764,7 +1786,8 @@ def deserialize_many_sparse(serialized_sparse, dtype, rank=None, name=None): return sparse_tensor.SparseTensor(output_indices, output_values, output_shape) -@tf_export("sparse_tensor_dense_matmul") +@tf_export("sparse.matmul", "sparse_tensor_dense_matmul") +@deprecation.deprecated_endpoints("sparse_tensor_dense_matmul") def sparse_tensor_dense_matmul(sp_a, b, adjoint_a=False, @@ -1777,7 +1800,7 @@ def sparse_tensor_dense_matmul(sp_a, following input format is recommended for optimal behavior: * If `adjoint_a == false`: `A` should be sorted in lexicographically - increasing order. Use `sparse_reorder` if you're not sure. + increasing order. Use `sparse.reorder` if you're not sure. * If `adjoint_a == true`: `A` should be sorted in order of increasing dimension 1 (i.e., "column major" order instead of "row major" order). @@ -1981,7 +2004,8 @@ def sparse_tensor_dense_matmul(sp_a, adjoint_b=adjoint_b) -@tf_export("sparse_softmax") +@tf_export("sparse.softmax", "sparse_softmax") +@deprecation.deprecated_endpoints("sparse_softmax") def sparse_softmax(sp_input, name=None): """Applies softmax to a batched N-D `SparseTensor`. @@ -2036,7 +2060,8 @@ def sparse_softmax(sp_input, name=None): sp_input.dense_shape) -@tf_export("sparse_maximum") +@tf_export("sparse.maximum", "sparse_maximum") +@deprecation.deprecated_endpoints("sparse_maximum") def sparse_maximum(sp_a, sp_b, name=None): """Returns the element-wise max of two SparseTensors. @@ -2073,7 +2098,8 @@ def sparse_maximum(sp_a, sp_b, name=None): return sparse_tensor.SparseTensor(out_indices, out_values, sp_a.dense_shape) -@tf_export("sparse_minimum") +@tf_export("sparse.minimum", "sparse_minimum") +@deprecation.deprecated_endpoints("sparse_minimum") def sparse_minimum(sp_a, sp_b, name=None): """Returns the element-wise min of two SparseTensors. @@ -2110,7 +2136,8 @@ def sparse_minimum(sp_a, sp_b, name=None): return sparse_tensor.SparseTensor(out_indices, out_values, sp_a.dense_shape) -@tf_export("sparse_transpose") +@tf_export("sparse.transpose", "sparse_transpose") +@deprecation.deprecated_endpoints("sparse_transpose") def sparse_transpose(sp_input, perm=None, name=None): """Transposes a `SparseTensor` @@ -2259,7 +2286,7 @@ def _take_many_sparse_from_tensors_map(sparse_map_op, The input `SparseTensor` objects' indices are assumed ordered in standard lexicographic order. If this is not the case, after this - step run `sparse_reorder` to restore index ordering. + step run `sparse.reorder` to restore index ordering. For example, if the serialized input is a `[2, 3]` matrix representing two original `SparseTensor` objects: diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py index 9a10abfcf7..cfab943896 100644 --- a/tensorflow/python/ops/special_math_ops.py +++ b/tensorflow/python/ops/special_math_ops.py @@ -29,11 +29,13 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export # TODO(b/27419586) Change docstring for required dtype of x once int allowed -@tf_export('lbeta') +@tf_export('math.lbeta', 'lbeta') +@deprecation.deprecated_endpoints('lbeta') def lbeta(x, name=None): r"""Computes \\(ln(|Beta(x)|)\\), reducing along the last dimension. diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py index 046a48d192..e83c08f643 100644 --- a/tensorflow/python/ops/string_ops.py +++ b/tensorflow/python/ops/string_ops.py @@ -310,8 +310,9 @@ def _reduce_join_reduction_dims(x, axis, reduction_indices): return math_ops.range(array_ops.rank(x) - 1, -1, -1) -@tf_export("reduce_join") -def reduce_join(inputs, axis=None, +@tf_export("strings.reduce_join", "reduce_join") +@deprecation.deprecated_endpoints("reduce_join") +def reduce_join(inputs, axis=None, # pylint: disable=missing-docstring keep_dims=False, separator="", name=None, @@ -329,6 +330,8 @@ def reduce_join(inputs, axis=None, reduce_join.__doc__ = deprecation.rewrite_argument_docstring( gen_string_ops.reduce_join.__doc__, "reduction_indices", "axis") +reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(", + "tf.strings.reduce_join(") # This wrapper provides backwards compatibility for code that predates the diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py index 8e7f123a85..8bf057f69d 100644 --- a/tensorflow/python/saved_model/builder_impl.py +++ b/tensorflow/python/saved_model/builder_impl.py @@ -36,10 +36,13 @@ from tensorflow.python.saved_model import utils_impl as saved_model_utils from tensorflow.python.training import saver as tf_saver from tensorflow.python.util import compat from tensorflow.python.util.deprecation import deprecated_args +from tensorflow.python.util.deprecation import deprecated_endpoints from tensorflow.python.util.tf_export import tf_export -@tf_export("saved_model.builder.SavedModelBuilder") +@tf_export("saved_model.Builder", + "saved_model.builder.SavedModelBuilder") +@deprecated_endpoints("saved_model.builder.SavedModelBuilder") class SavedModelBuilder(object): """Builds the `SavedModel` protocol buffer and saves variables and assets. @@ -61,7 +64,7 @@ class SavedModelBuilder(object): Typical usage for the `SavedModelBuilder`: ```python ... - builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + builder = tf.saved_model.Builder(export_dir) with tf.Session(graph=tf.Graph()) as sess: ... diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py index e8536108e8..895644a030 100644 --- a/tensorflow/python/saved_model/loader_impl.py +++ b/tensorflow/python/saved_model/loader_impl.py @@ -34,6 +34,7 @@ from tensorflow.python.saved_model import constants from tensorflow.python.saved_model import utils_impl as saved_model_utils from tensorflow.python.training import saver as tf_saver from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -144,7 +145,10 @@ def _get_main_op_tensor( return main_op_tensor -@tf_export("saved_model.loader.maybe_saved_model_directory") +@tf_export("saved_model.maybe_saved_model_directory", + "saved_model.loader.maybe_saved_model_directory") +@deprecation.deprecated_endpoints( + "saved_model.loader.maybe_saved_model_directory") def maybe_saved_model_directory(export_dir): """Checks whether the provided export directory could contain a SavedModel. @@ -165,7 +169,7 @@ def maybe_saved_model_directory(export_dir): return file_io.file_exists(txt_path) or file_io.file_exists(pb_path) -@tf_export("saved_model.loader.load") +@tf_export("saved_model.load", "saved_model.loader.load") def load(sess, tags, export_dir, import_scope=None, **saver_kwargs): """Loads the model from a SavedModel as specified by tags. diff --git a/tensorflow/python/saved_model/main_op_impl.py b/tensorflow/python/saved_model/main_op_impl.py index 631ee63729..ad4511b28e 100644 --- a/tensorflow/python/saved_model/main_op_impl.py +++ b/tensorflow/python/saved_model/main_op_impl.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import variables +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -42,7 +43,9 @@ def main_op(): # TODO(sukritiramesh): Integrate with Saver for complete restore functionality. -@tf_export('saved_model.main_op.main_op_with_restore') +@tf_export('saved_model.main_op_with_restore', + 'saved_model.main_op.main_op_with_restore') +@deprecation.deprecated_endpoints('saved_model.main_op.main_op_with_restore') def main_op_with_restore(restore_op_name): """Returns a main op to init variables, tables and restore the graph. diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py index 37f927f381..a1034416e9 100644 --- a/tensorflow/python/saved_model/signature_def_utils_impl.py +++ b/tensorflow/python/saved_model/signature_def_utils_impl.py @@ -24,10 +24,14 @@ from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.framework import ops from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import utils +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export('saved_model.signature_def_utils.build_signature_def') +@tf_export('saved_model.build_signature_def', + 'saved_model.signature_def_utils.build_signature_def') +@deprecation.deprecated_endpoints( + 'saved_model.signature_def_utils.build_signature_def') def build_signature_def(inputs=None, outputs=None, method_name=None): """Utility function to build a SignatureDef protocol buffer. @@ -53,7 +57,10 @@ def build_signature_def(inputs=None, outputs=None, method_name=None): return signature_def -@tf_export('saved_model.signature_def_utils.regression_signature_def') +@tf_export('saved_model.regression_signature_def', + 'saved_model.signature_def_utils.regression_signature_def') +@deprecation.deprecated_endpoints( + 'saved_model.signature_def_utils.regression_signature_def') def regression_signature_def(examples, predictions): """Creates regression signature from given examples and predictions. @@ -95,7 +102,10 @@ def regression_signature_def(examples, predictions): return signature_def -@tf_export('saved_model.signature_def_utils.classification_signature_def') +@tf_export('saved_model.classification_signature_def', + 'saved_model.signature_def_utils.classification_signature_def') +@deprecation.deprecated_endpoints( + 'saved_model.signature_def_utils.classification_signature_def') def classification_signature_def(examples, classes, scores): """Creates classification signature from given examples and predictions. @@ -148,7 +158,10 @@ def classification_signature_def(examples, classes, scores): return signature_def -@tf_export('saved_model.signature_def_utils.predict_signature_def') +@tf_export('saved_model.predict_signature_def', + 'saved_model.signature_def_utils.predict_signature_def') +@deprecation.deprecated_endpoints( + 'saved_model.signature_def_utils.predict_signature_def') def predict_signature_def(inputs, outputs): """Creates prediction signature from given inputs and outputs. @@ -239,7 +252,10 @@ def _supervised_signature_def( return signature_def -@tf_export('saved_model.signature_def_utils.is_valid_signature') +@tf_export('saved_model.is_valid_signature', + 'saved_model.signature_def_utils.is_valid_signature') +@deprecation.deprecated_endpoints( + 'saved_model.signature_def_utils.is_valid_signature') def is_valid_signature(signature_def): """Determine whether a SignatureDef can be served by TensorFlow Serving.""" if signature_def is None: @@ -313,4 +329,3 @@ def _is_valid_classification_signature(signature_def): return False return True - diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py index 06d09325c8..0bba7b6fac 100644 --- a/tensorflow/python/saved_model/utils_impl.py +++ b/tensorflow/python/saved_model/utils_impl.py @@ -27,13 +27,16 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.lib.io import file_io from tensorflow.python.saved_model import constants from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export # TensorInfo helpers. -@tf_export("saved_model.utils.build_tensor_info") +@tf_export("saved_model.build_tensor_info", + "saved_model.utils.build_tensor_info") +@deprecation.deprecated_endpoints("saved_model.utils.build_tensor_info") def build_tensor_info(tensor): """Utility function to build TensorInfo proto. @@ -57,7 +60,10 @@ def build_tensor_info(tensor): return tensor_info -@tf_export("saved_model.utils.get_tensor_from_tensor_info") +@tf_export("saved_model.get_tensor_from_tensor_info", + "saved_model.utils.get_tensor_from_tensor_info") +@deprecation.deprecated_endpoints( + "saved_model.utils.get_tensor_from_tensor_info") def get_tensor_from_tensor_info(tensor_info, graph=None, import_scope=None): """Returns the Tensor or SparseTensor described by a TensorInfo proto. diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl index 92446e2f8f..5ce5410e0b 100644 --- a/tensorflow/python/tools/api/generator/api_init_files.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files.bzl @@ -69,6 +69,7 @@ TENSORFLOW_API_INIT_FILES = [ "profiler/__init__.py", "python_io/__init__.py", "quantization/__init__.py", + "random/__init__.py", "resource_loader/__init__.py", "strings/__init__.py", "saved_model/__init__.py", diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl index bc2f3516d1..587eb232f5 100644 --- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl @@ -69,6 +69,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [ "profiler/__init__.py", "python_io/__init__.py", "quantization/__init__.py", + "random/__init__.py", "resource_loader/__init__.py", "strings/__init__.py", "saved_model/__init__.py", diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index 9d9db70890..eb131ac9f7 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -56,7 +56,8 @@ _restore_sparse = sparse_ops._take_many_sparse_from_tensors_map # pylint: enable=protected-access -@tf_export("train.match_filenames_once") +@tf_export("io.match_filenames_once", "train.match_filenames_once") +@deprecation.deprecated_endpoints("train.match_filenames_once") def match_filenames_once(pattern, name=None): """Save the list of files matching pattern, so it is only computed once. diff --git a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt index d9efe97821..ab6287f8cd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt @@ -1,5 +1,89 @@ path: "tensorflow.debugging" tf_module { + member_method { + name: "Assert" + argspec: "args=[\'condition\', \'data\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_all_finite" + argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "assert_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_greater" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_greater_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_integer" + argspec: "args=[\'x\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_less" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_less_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_near" + argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_negative" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_non_negative" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_non_positive" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_none_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_positive" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_proper_iterable" + argspec: "args=[\'values\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "assert_rank" + argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_rank_at_least" + argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_rank_in" + argspec: "args=[\'x\', \'ranks\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_same_float_dtype" + argspec: "args=[\'tensors\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_scalar" + argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "assert_type" + argspec: "args=[\'tensor\', \'tf_type\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "check_numerics" argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,4 +100,16 @@ tf_module { name: "is_nan" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "is_non_decreasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_numeric_tensor" + argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_strictly_increasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.dtypes.-d-type.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.-d-type.pbtxt new file mode 100644 index 0000000000..423eca32a2 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.-d-type.pbtxt @@ -0,0 +1,77 @@ +path: "tensorflow.dtypes.DType" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "as_datatype_enum" + mtype: "" + } + member { + name: "as_numpy_dtype" + mtype: "" + } + member { + name: "base_dtype" + mtype: "" + } + member { + name: "is_bool" + mtype: "" + } + member { + name: "is_complex" + mtype: "" + } + member { + name: "is_floating" + mtype: "" + } + member { + name: "is_integer" + mtype: "" + } + member { + name: "is_numpy_compatible" + mtype: "" + } + member { + name: "is_quantized" + mtype: "" + } + member { + name: "is_unsigned" + mtype: "" + } + member { + name: "limits" + mtype: "" + } + member { + name: "max" + mtype: "" + } + member { + name: "min" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "real_dtype" + mtype: "" + } + member { + name: "size" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'type_enum\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_compatible_with" + argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt index 98e1feed00..ea23feca84 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt @@ -1,7 +1,27 @@ path: "tensorflow.dtypes" tf_module { + member { + name: "DType" + mtype: "" + } + member_method { + name: "as_dtype" + argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "as_string" argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], " } + member_method { + name: "cast" + argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "complex" + argspec: "args=[\'real\', \'imag\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "saturate_cast" + argspec: "args=[\'value\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.graph_util.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.graph_util.pbtxt index eeabf845dc..162ee76ee7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.graph_util.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.graph_util.pbtxt @@ -8,6 +8,10 @@ tf_module { name: "extract_sub_graph" argspec: "args=[\'graph_def\', \'dest_nodes\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "import_graph_def" + argspec: "args=[\'graph_def\', \'input_map\', \'return_elements\', \'name\', \'op_dict\', \'producer_op_list\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "must_run_on_cpu" argspec: "args=[\'node\', \'pin_variables_on_cpu\'], varargs=None, keywords=None, defaults=[\'False\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt index 5c46dc5ee7..0a231f1b65 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt @@ -148,6 +148,10 @@ tf_module { name: "random_contrast" argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "random_crop" + argspec: "args=[\'value\', \'size\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "random_flip_left_right" argspec: "args=[\'image\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.initializers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.initializers.pbtxt index d499c67d89..19ca62122e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.initializers.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.initializers.pbtxt @@ -72,6 +72,10 @@ tf_module { name: "local_variables" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "tables_initializer" + argspec: "args=[\'name\'], varargs=None, keywords=None, defaults=[\'init_all_tables\'], " + } member_method { name: "variables" argspec: "args=[\'var_list\', \'name\'], varargs=None, keywords=None, defaults=[\'init\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-feature.pbtxt new file mode 100644 index 0000000000..cd0e51c8c7 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-feature.pbtxt @@ -0,0 +1,27 @@ +path: "tensorflow.io.FixedLenFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "default_value" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-sequence-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-sequence-feature.pbtxt new file mode 100644 index 0000000000..8a38f25fdf --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-fixed-len-sequence-feature.pbtxt @@ -0,0 +1,31 @@ +path: "tensorflow.io.FixedLenSequenceFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "allow_missing" + mtype: "" + } + member { + name: "default_value" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-padding-f-i-f-o-queue.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-padding-f-i-f-o-queue.pbtxt new file mode 100644 index 0000000000..85306fdcac --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-padding-f-i-f-o-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.PaddingFIFOQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'dtypes\', \'shapes\', \'names\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'padding_fifo_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-priority-queue.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-priority-queue.pbtxt new file mode 100644 index 0000000000..02d8037b34 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-priority-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.PriorityQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'types\', \'shapes\', \'names\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'priority_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-queue-base.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-queue-base.pbtxt new file mode 100644 index 0000000000..a30481a0ea --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-queue-base.pbtxt @@ -0,0 +1,65 @@ +path: "tensorflow.io.QueueBase" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtypes\', \'shapes\', \'names\', \'queue_ref\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-random-shuffle-queue.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-random-shuffle-queue.pbtxt new file mode 100644 index 0000000000..82cbf9884f --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-random-shuffle-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.RandomShuffleQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'min_after_dequeue\', \'dtypes\', \'shapes\', \'names\', \'seed\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'random_shuffle_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-sparse-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-sparse-feature.pbtxt new file mode 100644 index 0000000000..216947b4ed --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-sparse-feature.pbtxt @@ -0,0 +1,35 @@ +path: "tensorflow.io.SparseFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "already_sorted" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "index_key" + mtype: "" + } + member { + name: "size" + mtype: "" + } + member { + name: "value_key" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-compression-type.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-compression-type.pbtxt new file mode 100644 index 0000000000..b598f73d7e --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-compression-type.pbtxt @@ -0,0 +1,20 @@ +path: "tensorflow.io.TFRecordCompressionType" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "GZIP" + mtype: "" + } + member { + name: "NONE" + mtype: "" + } + member { + name: "ZLIB" + mtype: "" + } + member_method { + name: "__init__" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-options.pbtxt new file mode 100644 index 0000000000..bfbf37ccf4 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-options.pbtxt @@ -0,0 +1,17 @@ +path: "tensorflow.io.TFRecordOptions" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "compression_type_map" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'compression_type\', \'flush_mode\', \'input_buffer_size\', \'output_buffer_size\', \'window_bits\', \'compression_level\', \'compression_method\', \'mem_level\', \'compression_strategy\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "get_compression_type_string" + argspec: "args=[\'cls\', \'options\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-writer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-writer.pbtxt new file mode 100644 index 0000000000..6fd443f6d7 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-t-f-record-writer.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.io.TFRecordWriter" +tf_class { + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "flush" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "write" + argspec: "args=[\'self\', \'record\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.-var-len-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.-var-len-feature.pbtxt new file mode 100644 index 0000000000..fd835dbfbb --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.-var-len-feature.pbtxt @@ -0,0 +1,19 @@ +path: "tensorflow.io.VarLenFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtype" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt index 8938cf217b..dccf136788 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt @@ -1,5 +1,49 @@ path: "tensorflow.io" tf_module { + member { + name: "FixedLenFeature" + mtype: "" + } + member { + name: "FixedLenSequenceFeature" + mtype: "" + } + member { + name: "PaddingFIFOQueue" + mtype: "" + } + member { + name: "PriorityQueue" + mtype: "" + } + member { + name: "QueueBase" + mtype: "" + } + member { + name: "RandomShuffleQueue" + mtype: "" + } + member { + name: "SparseFeature" + mtype: "" + } + member { + name: "TFRecordCompressionType" + mtype: "" + } + member { + name: "TFRecordOptions" + mtype: "" + } + member { + name: "TFRecordWriter" + mtype: "" + } + member { + name: "VarLenFeature" + mtype: "" + } member_method { name: "decode_base64" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -8,6 +52,10 @@ tf_module { name: "decode_compressed" argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " } + member_method { + name: "decode_csv" + argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " + } member_method { name: "decode_json_example" argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,18 +64,38 @@ tf_module { name: "decode_raw" argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " } + member_method { + name: "deserialize_many_sparse" + argspec: "args=[\'serialized_sparse\', \'dtype\', \'rank\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "encode_base64" argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "match_filenames_once" + argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "matching_files" argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "parse_example" + argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "parse_sequence_example" argspec: "args=[\'serialized\', \'context_features\', \'sequence_features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "parse_single_example" + argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "parse_single_sequence_example" + argspec: "args=[\'serialized\', \'context_features\', \'sequence_features\', \'example_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "parse_tensor" argspec: "args=[\'serialized\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -36,8 +104,24 @@ tf_module { name: "read_file" argspec: "args=[\'filename\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "serialize_many_sparse" + argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + } + member_method { + name: "serialize_sparse" + argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + } + member_method { + name: "tf_record_iterator" + argspec: "args=[\'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "write_file" argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "write_graph" + argspec: "args=[\'graph_or_graph_def\', \'logdir\', \'name\', \'as_text\'], varargs=None, keywords=None, defaults=[\'True\'], " + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt index d979116887..6ac95d96da 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt @@ -108,10 +108,18 @@ tf_module { name: "eye" argspec: "args=[\'num_rows\', \'num_columns\', \'batch_shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\'], " } + member_method { + name: "global_norm" + argspec: "args=[\'t_list\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "inv" argspec: "args=[\'input\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "l2_normalize" + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + } member_method { name: "logdet" argspec: "args=[\'matrix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -124,6 +132,10 @@ tf_module { name: "lstsq" argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'True\', \'None\'], " } + member_method { + name: "matmul" + argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'adjoint_a\', \'adjoint_b\', \'a_is_sparse\', \'b_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'False\', \'None\'], " + } member_method { name: "norm" argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt index 72856466ec..459b9e3684 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt @@ -1,5 +1,13 @@ path: "tensorflow.math" tf_module { + member_method { + name: "abs" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "accumulate_n" + argspec: "args=[\'inputs\', \'shape\', \'tensor_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "acos" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -12,6 +20,22 @@ tf_module { name: "add" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_n" + argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "angle" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "argmax" + argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } + member_method { + name: "argmin" + argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } member_method { name: "asin" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -52,10 +76,18 @@ tf_module { name: "betainc" argspec: "args=[\'a\', \'b\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "bincount" + argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } member_method { name: "ceil" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "conj" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "cos" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -64,14 +96,34 @@ tf_module { name: "cosh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "count_nonzero" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\', \'None\', \'None\'], " + } + member_method { + name: "cumprod" + argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " + } + member_method { + name: "cumsum" + argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " + } member_method { name: "digamma" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "divide" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "erf" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "erfc" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -88,6 +140,10 @@ tf_module { name: "floor" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "floordiv" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "greater" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +160,26 @@ tf_module { name: "igammac" argspec: "args=[\'a\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "imag" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "in_top_k" + argspec: "args=[\'predictions\', \'targets\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "invert_permutation" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "l2_normalize" + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + } + member_method { + name: "lbeta" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "less" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -128,6 +200,14 @@ tf_module { name: "log1p" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "log_sigmoid" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "log_softmax" + argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "logical_and" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -140,6 +220,10 @@ tf_module { name: "logical_or" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "logical_xor" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'LogicalXor\'], " + } member_method { name: "maximum" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -148,6 +232,14 @@ tf_module { name: "minimum" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "multiply" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "negative" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "not_equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -160,18 +252,66 @@ tf_module { name: "polyval" argspec: "args=[\'coeffs\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "pow" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "real" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "reciprocal" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "reduce_all" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_any" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_logsumexp" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_max" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_mean" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_min" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_prod" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "rint" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "round" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "rsqrt" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "scalar_mul" + argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "segment_max" argspec: "args=[\'data\', \'segment_ids\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -192,6 +332,14 @@ tf_module { name: "segment_sum" argspec: "args=[\'data\', \'segment_ids\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sigmoid" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "sign" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "sin" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -200,6 +348,10 @@ tf_module { name: "sinh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "softmax" + argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "softplus" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -208,18 +360,46 @@ tf_module { name: "softsign" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sqrt" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "square" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "squared_difference" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "subtract" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "tan" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "tanh" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "top_k" + argspec: "args=[\'input\', \'k\', \'sorted\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'True\', \'None\'], " + } + member_method { + name: "truediv" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_max" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "unsorted_segment_mean" + argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_min" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -228,6 +408,10 @@ tf_module { name: "unsorted_segment_prod" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "unsorted_segment_sqrt_n" + argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_sum" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -240,6 +424,10 @@ tf_module { name: "xlogy" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "zero_fraction" + argspec: "args=[\'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "zeta" argspec: "args=[\'x\', \'q\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt index d9e5b0d0fc..9b28ce5746 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt @@ -100,6 +100,10 @@ tf_module { name: "ctc_loss" argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], " } + member_method { + name: "depth_to_space" + argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], " + } member_method { name: "depthwise_conv2d" argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " @@ -304,6 +308,14 @@ tf_module { name: "softsign" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "space_to_batch" + argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "space_to_depth" + argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], " + } member_method { name: "sparse_softmax_cross_entropy_with_logits" argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 509ceff9df..a268529c1f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -496,6 +496,10 @@ tf_module { name: "quint8" mtype: "" } + member { + name: "random" + mtype: "" + } member { name: "random_normal_initializer" mtype: "" @@ -1744,6 +1748,10 @@ tf_module { name: "rint" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "roll" + argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "round" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt index 6d865efed0..77c92aeb0d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt @@ -28,6 +28,10 @@ tf_module { name: "fake_quant_with_min_max_vars_per_channel_gradient" argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\', \'name\'], varargs=None, keywords=None, defaults=[\'8\', \'False\', \'None\'], " } + member_method { + name: "quantize" + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'HALF_AWAY_FROM_ZERO\', \'None\'], " + } member_method { name: "quantized_concat" argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt new file mode 100644 index 0000000000..a568dd4cd8 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt @@ -0,0 +1,47 @@ +path: "tensorflow.random" +tf_module { + member_method { + name: "gamma" + argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " + } + member_method { + name: "get_seed" + argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "log_uniform_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "multinomial" + argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "normal" + argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " + } + member_method { + name: "poisson" + argspec: "args=[\'lam\', \'shape\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\'], " + } + member_method { + name: "set_random_seed" + argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "shuffle" + argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "truncated_normal" + argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " + } + member_method { + name: "uniform" + argspec: "args=[\'shape\', \'minval\', \'maxval\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \"\", \'None\', \'None\'], " + } + member_method { + name: "uniform_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt new file mode 100644 index 0000000000..67457de070 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.saved_model.Builder" +tf_class { + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'export_dir\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "add_meta_graph" + argspec: "args=[\'self\', \'tags\', \'signature_def_map\', \'assets_collection\', \'legacy_init_op\', \'clear_devices\', \'main_op\', \'strip_default_attrs\', \'saver\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'None\', \'False\', \'None\'], " + } + member_method { + name: "add_meta_graph_and_variables" + argspec: "args=[\'self\', \'sess\', \'tags\', \'signature_def_map\', \'assets_collection\', \'legacy_init_op\', \'clear_devices\', \'main_op\', \'strip_default_attrs\', \'saver\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'None\', \'False\', \'None\'], " + } + member_method { + name: "save" + argspec: "args=[\'self\', \'as_text\'], varargs=None, keywords=None, defaults=[\'False\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt index e1a0385092..3f4965fc69 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.saved_model" tf_module { + member { + name: "Builder" + mtype: "" + } member { name: "builder" mtype: "" @@ -32,6 +36,46 @@ tf_module { name: "utils" mtype: "" } + member_method { + name: "build_signature_def" + argspec: "args=[\'inputs\', \'outputs\', \'method_name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "build_tensor_info" + argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "classification_signature_def" + argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_tensor_from_tensor_info" + argspec: "args=[\'tensor_info\', \'graph\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "is_valid_signature" + argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "load" + argspec: "args=[\'sess\', \'tags\', \'export_dir\', \'import_scope\'], varargs=None, keywords=saver_kwargs, defaults=[\'None\'], " + } + member_method { + name: "main_op_with_restore" + argspec: "args=[\'restore_op_name\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "maybe_saved_model_directory" + argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "predict_signature_def" + argspec: "args=[\'inputs\', \'outputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "regression_signature_def" + argspec: "args=[\'examples\', \'predictions\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "simple_save" argspec: "args=[\'session\', \'export_dir\', \'inputs\', \'outputs\', \'legacy_init_op\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt new file mode 100644 index 0000000000..cd97716c9d --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt @@ -0,0 +1,46 @@ +path: "tensorflow.sparse.SparseConditionalAccumulator" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "accumulator_ref" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtype\', \'shape\', \'shared_name\', \'name\', \'reduction_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'sparse_conditional_accumulator\', \'MEAN\'], " + } + member_method { + name: "apply_grad" + argspec: "args=[\'self\', \'grad_indices\', \'grad_values\', \'grad_shape\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " + } + member_method { + name: "apply_indexed_slices_grad" + argspec: "args=[\'self\', \'grad\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + } + member_method { + name: "num_accumulated" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "set_global_step" + argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "take_grad" + argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "take_indexed_slices_grad" + argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt new file mode 100644 index 0000000000..02e59a63e1 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt @@ -0,0 +1,54 @@ +path: "tensorflow.sparse.SparseTensor" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dense_shape" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "graph" + mtype: "" + } + member { + name: "indices" + mtype: "" + } + member { + name: "op" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member { + name: "values" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "consumers" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "eval" + argspec: "args=[\'self\', \'feed_dict\', \'session\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "from_value" + argspec: "args=[\'cls\', \'sparse_tensor_value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_shape" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt index ba9e651b34..32bd8d5f8e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt @@ -1,5 +1,21 @@ path: "tensorflow.sparse" tf_module { + member { + name: "SparseConditionalAccumulator" + mtype: "" + } + member { + name: "SparseTensor" + mtype: "" + } + member_method { + name: "add" + argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], " + } + member_method { + name: "concat" + argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "cross" argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,4 +32,100 @@ tf_module { name: "eye" argspec: "args=[\'num_rows\', \'num_columns\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " } + member_method { + name: "fill_empty_rows" + argspec: "args=[\'sp_input\', \'default_value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "mask" + argspec: "args=[\'a\', \'mask_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "matmul" + argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], " + } + member_method { + name: "maximum" + argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "merge" + argspec: "args=[\'sp_ids\', \'sp_values\', \'vocab_size\', \'name\', \'already_sorted\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " + } + member_method { + name: "minimum" + argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "placeholder" + argspec: "args=[\'dtype\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "reduce_max" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_max_sparse" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum_sparse" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reorder" + argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "reset_shape" + argspec: "args=[\'sp_input\', \'new_shape\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "reshape" + argspec: "args=[\'sp_input\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "retain" + argspec: "args=[\'sp_input\', \'to_retain\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "segment_mean" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "segment_sqrt_n" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "segment_sum" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "slice" + argspec: "args=[\'sp_input\', \'start\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "softmax" + argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "split" + argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "to_dense" + argspec: "args=[\'sp_input\', \'default_value\', \'validate_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'True\', \'None\'], " + } + member_method { + name: "to_indicator" + argspec: "args=[\'sp_input\', \'vocab_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "transpose" + argspec: "args=[\'sp_input\', \'perm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt index 312e94b41d..ebdaf57231 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt @@ -12,6 +12,10 @@ tf_module { name: "length" argspec: "args=[\'input\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], " } + member_method { + name: "reduce_join" + argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], " + } member_method { name: "regex_full_match" argspec: "args=[\'input\', \'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt index 9f35395284..45c81fdd3b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt @@ -272,6 +272,10 @@ tf_module { name: "checkpoint_exists" argspec: "args=[\'checkpoint_prefix\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "confusion_matrix" + argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'dtype\', \'name\', \'weights\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " + } member_method { name: "cosine_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt index d9efe97821..ab6287f8cd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt @@ -1,5 +1,89 @@ path: "tensorflow.debugging" tf_module { + member_method { + name: "Assert" + argspec: "args=[\'condition\', \'data\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_all_finite" + argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "assert_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_greater" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_greater_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_integer" + argspec: "args=[\'x\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_less" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_less_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_near" + argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_negative" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_non_negative" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_non_positive" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_none_equal" + argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_positive" + argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_proper_iterable" + argspec: "args=[\'values\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "assert_rank" + argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_rank_at_least" + argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_rank_in" + argspec: "args=[\'x\', \'ranks\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "assert_same_float_dtype" + argspec: "args=[\'tensors\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "assert_scalar" + argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "assert_type" + argspec: "args=[\'tensor\', \'tf_type\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "check_numerics" argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,4 +100,16 @@ tf_module { name: "is_nan" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "is_non_decreasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_numeric_tensor" + argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_strictly_increasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.-d-type.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.-d-type.pbtxt new file mode 100644 index 0000000000..423eca32a2 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.-d-type.pbtxt @@ -0,0 +1,77 @@ +path: "tensorflow.dtypes.DType" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "as_datatype_enum" + mtype: "" + } + member { + name: "as_numpy_dtype" + mtype: "" + } + member { + name: "base_dtype" + mtype: "" + } + member { + name: "is_bool" + mtype: "" + } + member { + name: "is_complex" + mtype: "" + } + member { + name: "is_floating" + mtype: "" + } + member { + name: "is_integer" + mtype: "" + } + member { + name: "is_numpy_compatible" + mtype: "" + } + member { + name: "is_quantized" + mtype: "" + } + member { + name: "is_unsigned" + mtype: "" + } + member { + name: "limits" + mtype: "" + } + member { + name: "max" + mtype: "" + } + member { + name: "min" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "real_dtype" + mtype: "" + } + member { + name: "size" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'type_enum\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_compatible_with" + argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt index 98e1feed00..ea23feca84 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt @@ -1,7 +1,27 @@ path: "tensorflow.dtypes" tf_module { + member { + name: "DType" + mtype: "" + } + member_method { + name: "as_dtype" + argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "as_string" argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], " } + member_method { + name: "cast" + argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "complex" + argspec: "args=[\'real\', \'imag\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "saturate_cast" + argspec: "args=[\'value\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.graph_util.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.graph_util.pbtxt index eeabf845dc..162ee76ee7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.graph_util.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.graph_util.pbtxt @@ -8,6 +8,10 @@ tf_module { name: "extract_sub_graph" argspec: "args=[\'graph_def\', \'dest_nodes\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "import_graph_def" + argspec: "args=[\'graph_def\', \'input_map\', \'return_elements\', \'name\', \'op_dict\', \'producer_op_list\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "must_run_on_cpu" argspec: "args=[\'node\', \'pin_variables_on_cpu\'], varargs=None, keywords=None, defaults=[\'False\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt index 5c46dc5ee7..0a231f1b65 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt @@ -148,6 +148,10 @@ tf_module { name: "random_contrast" argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "random_crop" + argspec: "args=[\'value\', \'size\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "random_flip_left_right" argspec: "args=[\'image\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.initializers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.initializers.pbtxt index e3c63fe737..d49181714f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.initializers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.initializers.pbtxt @@ -64,4 +64,8 @@ tf_module { name: "lecun_uniform" argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "tables_initializer" + argspec: "args=[\'name\'], varargs=None, keywords=None, defaults=[\'init_all_tables\'], " + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-feature.pbtxt new file mode 100644 index 0000000000..cd0e51c8c7 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-feature.pbtxt @@ -0,0 +1,27 @@ +path: "tensorflow.io.FixedLenFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "default_value" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-sequence-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-sequence-feature.pbtxt new file mode 100644 index 0000000000..8a38f25fdf --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-fixed-len-sequence-feature.pbtxt @@ -0,0 +1,31 @@ +path: "tensorflow.io.FixedLenSequenceFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "allow_missing" + mtype: "" + } + member { + name: "default_value" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-padding-f-i-f-o-queue.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-padding-f-i-f-o-queue.pbtxt new file mode 100644 index 0000000000..85306fdcac --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-padding-f-i-f-o-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.PaddingFIFOQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'dtypes\', \'shapes\', \'names\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'padding_fifo_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-priority-queue.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-priority-queue.pbtxt new file mode 100644 index 0000000000..02d8037b34 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-priority-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.PriorityQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'types\', \'shapes\', \'names\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'priority_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-queue-base.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-queue-base.pbtxt new file mode 100644 index 0000000000..a30481a0ea --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-queue-base.pbtxt @@ -0,0 +1,65 @@ +path: "tensorflow.io.QueueBase" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtypes\', \'shapes\', \'names\', \'queue_ref\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-random-shuffle-queue.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-random-shuffle-queue.pbtxt new file mode 100644 index 0000000000..82cbf9884f --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-random-shuffle-queue.pbtxt @@ -0,0 +1,66 @@ +path: "tensorflow.io.RandomShuffleQueue" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtypes" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "names" + mtype: "" + } + member { + name: "queue_ref" + mtype: "" + } + member { + name: "shapes" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'capacity\', \'min_after_dequeue\', \'dtypes\', \'shapes\', \'names\', \'seed\', \'shared_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'random_shuffle_queue\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\', \'cancel_pending_enqueues\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + } + member_method { + name: "dequeue" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_many" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "dequeue_up_to" + argspec: "args=[\'self\', \'n\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "enqueue_many" + argspec: "args=[\'self\', \'vals\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_list" + argspec: "args=[\'index\', \'queues\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "is_closed" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "size" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-sparse-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-sparse-feature.pbtxt new file mode 100644 index 0000000000..216947b4ed --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-sparse-feature.pbtxt @@ -0,0 +1,35 @@ +path: "tensorflow.io.SparseFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "already_sorted" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "index_key" + mtype: "" + } + member { + name: "size" + mtype: "" + } + member { + name: "value_key" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-compression-type.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-compression-type.pbtxt new file mode 100644 index 0000000000..b598f73d7e --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-compression-type.pbtxt @@ -0,0 +1,20 @@ +path: "tensorflow.io.TFRecordCompressionType" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "GZIP" + mtype: "" + } + member { + name: "NONE" + mtype: "" + } + member { + name: "ZLIB" + mtype: "" + } + member_method { + name: "__init__" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-options.pbtxt new file mode 100644 index 0000000000..bfbf37ccf4 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-options.pbtxt @@ -0,0 +1,17 @@ +path: "tensorflow.io.TFRecordOptions" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "compression_type_map" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'compression_type\', \'flush_mode\', \'input_buffer_size\', \'output_buffer_size\', \'window_bits\', \'compression_level\', \'compression_method\', \'mem_level\', \'compression_strategy\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "get_compression_type_string" + argspec: "args=[\'cls\', \'options\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-writer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-writer.pbtxt new file mode 100644 index 0000000000..6fd443f6d7 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-t-f-record-writer.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.io.TFRecordWriter" +tf_class { + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "close" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "flush" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "write" + argspec: "args=[\'self\', \'record\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.-var-len-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.-var-len-feature.pbtxt new file mode 100644 index 0000000000..fd835dbfbb --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.-var-len-feature.pbtxt @@ -0,0 +1,19 @@ +path: "tensorflow.io.VarLenFeature" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dtype" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "count" + } + member_method { + name: "index" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt index 8938cf217b..dccf136788 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt @@ -1,5 +1,49 @@ path: "tensorflow.io" tf_module { + member { + name: "FixedLenFeature" + mtype: "" + } + member { + name: "FixedLenSequenceFeature" + mtype: "" + } + member { + name: "PaddingFIFOQueue" + mtype: "" + } + member { + name: "PriorityQueue" + mtype: "" + } + member { + name: "QueueBase" + mtype: "" + } + member { + name: "RandomShuffleQueue" + mtype: "" + } + member { + name: "SparseFeature" + mtype: "" + } + member { + name: "TFRecordCompressionType" + mtype: "" + } + member { + name: "TFRecordOptions" + mtype: "" + } + member { + name: "TFRecordWriter" + mtype: "" + } + member { + name: "VarLenFeature" + mtype: "" + } member_method { name: "decode_base64" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -8,6 +52,10 @@ tf_module { name: "decode_compressed" argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " } + member_method { + name: "decode_csv" + argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " + } member_method { name: "decode_json_example" argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,18 +64,38 @@ tf_module { name: "decode_raw" argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " } + member_method { + name: "deserialize_many_sparse" + argspec: "args=[\'serialized_sparse\', \'dtype\', \'rank\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "encode_base64" argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "match_filenames_once" + argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "matching_files" argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "parse_example" + argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "parse_sequence_example" argspec: "args=[\'serialized\', \'context_features\', \'sequence_features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "parse_single_example" + argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "parse_single_sequence_example" + argspec: "args=[\'serialized\', \'context_features\', \'sequence_features\', \'example_name\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "parse_tensor" argspec: "args=[\'serialized\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -36,8 +104,24 @@ tf_module { name: "read_file" argspec: "args=[\'filename\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "serialize_many_sparse" + argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + } + member_method { + name: "serialize_sparse" + argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + } + member_method { + name: "tf_record_iterator" + argspec: "args=[\'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "write_file" argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "write_graph" + argspec: "args=[\'graph_or_graph_def\', \'logdir\', \'name\', \'as_text\'], varargs=None, keywords=None, defaults=[\'True\'], " + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt index d979116887..6ac95d96da 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt @@ -108,10 +108,18 @@ tf_module { name: "eye" argspec: "args=[\'num_rows\', \'num_columns\', \'batch_shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\'], " } + member_method { + name: "global_norm" + argspec: "args=[\'t_list\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "inv" argspec: "args=[\'input\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "l2_normalize" + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + } member_method { name: "logdet" argspec: "args=[\'matrix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -124,6 +132,10 @@ tf_module { name: "lstsq" argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'True\', \'None\'], " } + member_method { + name: "matmul" + argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'adjoint_a\', \'adjoint_b\', \'a_is_sparse\', \'b_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'False\', \'None\'], " + } member_method { name: "norm" argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt index 72856466ec..459b9e3684 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt @@ -1,5 +1,13 @@ path: "tensorflow.math" tf_module { + member_method { + name: "abs" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "accumulate_n" + argspec: "args=[\'inputs\', \'shape\', \'tensor_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "acos" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -12,6 +20,22 @@ tf_module { name: "add" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_n" + argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "angle" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "argmax" + argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } + member_method { + name: "argmin" + argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } member_method { name: "asin" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -52,10 +76,18 @@ tf_module { name: "betainc" argspec: "args=[\'a\', \'b\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "bincount" + argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + } member_method { name: "ceil" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "conj" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "cos" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -64,14 +96,34 @@ tf_module { name: "cosh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "count_nonzero" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\', \'None\', \'None\'], " + } + member_method { + name: "cumprod" + argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " + } + member_method { + name: "cumsum" + argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " + } member_method { name: "digamma" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "divide" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "erf" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "erfc" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -88,6 +140,10 @@ tf_module { name: "floor" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "floordiv" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "greater" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +160,26 @@ tf_module { name: "igammac" argspec: "args=[\'a\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "imag" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "in_top_k" + argspec: "args=[\'predictions\', \'targets\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "invert_permutation" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "l2_normalize" + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + } + member_method { + name: "lbeta" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "less" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -128,6 +200,14 @@ tf_module { name: "log1p" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "log_sigmoid" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "log_softmax" + argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "logical_and" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -140,6 +220,10 @@ tf_module { name: "logical_or" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "logical_xor" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'LogicalXor\'], " + } member_method { name: "maximum" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -148,6 +232,14 @@ tf_module { name: "minimum" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "multiply" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "negative" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "not_equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -160,18 +252,66 @@ tf_module { name: "polyval" argspec: "args=[\'coeffs\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "pow" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "real" + argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "reciprocal" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "reduce_all" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_any" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_logsumexp" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_max" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_mean" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_min" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_prod" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum" + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "rint" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "round" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "rsqrt" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "scalar_mul" + argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "segment_max" argspec: "args=[\'data\', \'segment_ids\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -192,6 +332,14 @@ tf_module { name: "segment_sum" argspec: "args=[\'data\', \'segment_ids\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sigmoid" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "sign" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "sin" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -200,6 +348,10 @@ tf_module { name: "sinh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "softmax" + argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "softplus" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -208,18 +360,46 @@ tf_module { name: "softsign" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sqrt" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "square" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "squared_difference" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "subtract" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "tan" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "tanh" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "top_k" + argspec: "args=[\'input\', \'k\', \'sorted\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'True\', \'None\'], " + } + member_method { + name: "truediv" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_max" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "unsorted_segment_mean" + argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_min" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -228,6 +408,10 @@ tf_module { name: "unsorted_segment_prod" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "unsorted_segment_sqrt_n" + argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "unsorted_segment_sum" argspec: "args=[\'data\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -240,6 +424,10 @@ tf_module { name: "xlogy" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "zero_fraction" + argspec: "args=[\'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "zeta" argspec: "args=[\'x\', \'q\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt index d9e5b0d0fc..9b28ce5746 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt @@ -100,6 +100,10 @@ tf_module { name: "ctc_loss" argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], " } + member_method { + name: "depth_to_space" + argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], " + } member_method { name: "depthwise_conv2d" argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " @@ -304,6 +308,14 @@ tf_module { name: "softsign" argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "space_to_batch" + argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "space_to_depth" + argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], " + } member_method { name: "sparse_softmax_cross_entropy_with_logits" argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt index d2dc8bc85f..5b3ea75bce 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt @@ -456,6 +456,10 @@ tf_module { name: "quint8" mtype: "" } + member { + name: "random" + mtype: "" + } member { name: "random_normal_initializer" mtype: "" @@ -1608,6 +1612,10 @@ tf_module { name: "rint" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "roll" + argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "round" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt index 6d865efed0..77c92aeb0d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt @@ -28,6 +28,10 @@ tf_module { name: "fake_quant_with_min_max_vars_per_channel_gradient" argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\', \'name\'], varargs=None, keywords=None, defaults=[\'8\', \'False\', \'None\'], " } + member_method { + name: "quantize" + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'HALF_AWAY_FROM_ZERO\', \'None\'], " + } member_method { name: "quantized_concat" argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt new file mode 100644 index 0000000000..a568dd4cd8 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt @@ -0,0 +1,47 @@ +path: "tensorflow.random" +tf_module { + member_method { + name: "gamma" + argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " + } + member_method { + name: "get_seed" + argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "log_uniform_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "multinomial" + argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "normal" + argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " + } + member_method { + name: "poisson" + argspec: "args=[\'lam\', \'shape\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\'], " + } + member_method { + name: "set_random_seed" + argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "shuffle" + argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "truncated_normal" + argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " + } + member_method { + name: "uniform" + argspec: "args=[\'shape\', \'minval\', \'maxval\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \"\", \'None\', \'None\'], " + } + member_method { + name: "uniform_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.-builder.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.-builder.pbtxt new file mode 100644 index 0000000000..67457de070 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.-builder.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.saved_model.Builder" +tf_class { + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'export_dir\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "add_meta_graph" + argspec: "args=[\'self\', \'tags\', \'signature_def_map\', \'assets_collection\', \'legacy_init_op\', \'clear_devices\', \'main_op\', \'strip_default_attrs\', \'saver\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'None\', \'False\', \'None\'], " + } + member_method { + name: "add_meta_graph_and_variables" + argspec: "args=[\'self\', \'sess\', \'tags\', \'signature_def_map\', \'assets_collection\', \'legacy_init_op\', \'clear_devices\', \'main_op\', \'strip_default_attrs\', \'saver\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'None\', \'False\', \'None\'], " + } + member_method { + name: "save" + argspec: "args=[\'self\', \'as_text\'], varargs=None, keywords=None, defaults=[\'False\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt index e1a0385092..3f4965fc69 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.saved_model" tf_module { + member { + name: "Builder" + mtype: "" + } member { name: "builder" mtype: "" @@ -32,6 +36,46 @@ tf_module { name: "utils" mtype: "" } + member_method { + name: "build_signature_def" + argspec: "args=[\'inputs\', \'outputs\', \'method_name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "build_tensor_info" + argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "classification_signature_def" + argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_tensor_from_tensor_info" + argspec: "args=[\'tensor_info\', \'graph\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "is_valid_signature" + argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "load" + argspec: "args=[\'sess\', \'tags\', \'export_dir\', \'import_scope\'], varargs=None, keywords=saver_kwargs, defaults=[\'None\'], " + } + member_method { + name: "main_op_with_restore" + argspec: "args=[\'restore_op_name\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "maybe_saved_model_directory" + argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "predict_signature_def" + argspec: "args=[\'inputs\', \'outputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "regression_signature_def" + argspec: "args=[\'examples\', \'predictions\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "simple_save" argspec: "args=[\'session\', \'export_dir\', \'inputs\', \'outputs\', \'legacy_init_op\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt new file mode 100644 index 0000000000..cd97716c9d --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-conditional-accumulator.pbtxt @@ -0,0 +1,46 @@ +path: "tensorflow.sparse.SparseConditionalAccumulator" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "accumulator_ref" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtype\', \'shape\', \'shared_name\', \'name\', \'reduction_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'sparse_conditional_accumulator\', \'MEAN\'], " + } + member_method { + name: "apply_grad" + argspec: "args=[\'self\', \'grad_indices\', \'grad_values\', \'grad_shape\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " + } + member_method { + name: "apply_indexed_slices_grad" + argspec: "args=[\'self\', \'grad\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + } + member_method { + name: "num_accumulated" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "set_global_step" + argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "take_grad" + argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "take_indexed_slices_grad" + argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt new file mode 100644 index 0000000000..02e59a63e1 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt @@ -0,0 +1,54 @@ +path: "tensorflow.sparse.SparseTensor" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "dense_shape" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "graph" + mtype: "" + } + member { + name: "indices" + mtype: "" + } + member { + name: "op" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member { + name: "values" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "consumers" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "eval" + argspec: "args=[\'self\', \'feed_dict\', \'session\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "from_value" + argspec: "args=[\'cls\', \'sparse_tensor_value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_shape" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt index ba9e651b34..32bd8d5f8e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt @@ -1,5 +1,21 @@ path: "tensorflow.sparse" tf_module { + member { + name: "SparseConditionalAccumulator" + mtype: "" + } + member { + name: "SparseTensor" + mtype: "" + } + member_method { + name: "add" + argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], " + } + member_method { + name: "concat" + argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "cross" argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -16,4 +32,100 @@ tf_module { name: "eye" argspec: "args=[\'num_rows\', \'num_columns\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " } + member_method { + name: "fill_empty_rows" + argspec: "args=[\'sp_input\', \'default_value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "mask" + argspec: "args=[\'a\', \'mask_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "matmul" + argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], " + } + member_method { + name: "maximum" + argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "merge" + argspec: "args=[\'sp_ids\', \'sp_values\', \'vocab_size\', \'name\', \'already_sorted\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " + } + member_method { + name: "minimum" + argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "placeholder" + argspec: "args=[\'dtype\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "reduce_max" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_max_sparse" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reduce_sum_sparse" + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "reorder" + argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "reset_shape" + argspec: "args=[\'sp_input\', \'new_shape\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "reshape" + argspec: "args=[\'sp_input\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "retain" + argspec: "args=[\'sp_input\', \'to_retain\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "segment_mean" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "segment_sqrt_n" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "segment_sum" + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "slice" + argspec: "args=[\'sp_input\', \'start\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "softmax" + argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "split" + argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "to_dense" + argspec: "args=[\'sp_input\', \'default_value\', \'validate_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'True\', \'None\'], " + } + member_method { + name: "to_indicator" + argspec: "args=[\'sp_input\', \'vocab_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "transpose" + argspec: "args=[\'sp_input\', \'perm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt index 312e94b41d..ebdaf57231 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt @@ -12,6 +12,10 @@ tf_module { name: "length" argspec: "args=[\'input\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], " } + member_method { + name: "reduce_join" + argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], " + } member_method { name: "regex_full_match" argspec: "args=[\'input\', \'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt index cb6da5088b..7e980fe44d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt @@ -252,6 +252,10 @@ tf_module { name: "checkpoint_exists" argspec: "args=[\'checkpoint_prefix\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "confusion_matrix" + argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'dtype\', \'name\', \'weights\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " + } member_method { name: "cosine_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], " -- GitLab From 694367b574dcaf5ac90f3e42b8dee8fa51ca9f38 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 11:58:17 -0700 Subject: [PATCH 0160/1085] Automated rollback of commit cb98ceba9cff8c10ee3c7e89dc8925c88b28118e PiperOrigin-RevId: 215254762 --- tensorflow/core/grappler/optimizers/meta_optimizer.cc | 4 ++-- tensorflow/core/protobuf/rewriter_config.proto | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index a5f851fb1a..c3d70a1fdf 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -139,7 +139,7 @@ Status MetaOptimizer::InitializeOptimizers( if (cfg_.remapping() != RewriterConfig::OFF) { optimizers->push_back(MakeUnique(cfg_.remapping())); } - if (cfg_.pin_to_host_optimization() != RewriterConfig::OFF) { + if (cfg_.pin_to_host_optimization() == RewriterConfig::ON) { optimizers->push_back(MakeUnique()); } if (cfg_.arithmetic_optimization() != RewriterConfig::OFF) { @@ -527,7 +527,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) { cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT || cfg.debug_stripper() == RewriterConfig::ON || cfg.scoped_allocator_optimization() == RewriterConfig::ON || - cfg.pin_to_host_optimization() != RewriterConfig::OFF || + cfg.pin_to_host_optimization() == RewriterConfig::ON || !cfg.optimizers().empty() || !cfg.custom_optimizers().empty(); } diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto index 8e0448d536..8c31468ff5 100644 --- a/tensorflow/core/protobuf/rewriter_config.proto +++ b/tensorflow/core/protobuf/rewriter_config.proto @@ -75,7 +75,7 @@ message RewriterConfig { // Try to allocate some independent Op outputs contiguously in order to // merge or eliminate downstream Ops (off by default). Toggle scoped_allocator_optimization = 15; - // Force small ops onto the CPU (default is ON). + // Force small ops onto the CPU (default is OFF). Toggle pin_to_host_optimization = 18; // Disable the entire meta optimizer (off by default). bool disable_meta_optimizer = 19; -- GitLab From c4b3ce081b8abfae5560814ec445f0169cb4c368 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 1 Oct 2018 12:03:53 -0700 Subject: [PATCH 0161/1085] Add new attributes for the defun forward/backward functions. PiperOrigin-RevId: 215255826 --- tensorflow/python/eager/function.py | 39 ++++++++++++++++++------ tensorflow/python/eager/function_test.py | 15 +++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index dd3e1a3723..60a4f018cd 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -21,6 +21,7 @@ from __future__ import print_function import collections import functools +import re import sys import threading import weakref @@ -61,9 +62,15 @@ cond_v2_impl._function = sys.modules[__name__] # pylint: disable=protected-acce # This is to avoid a circular dependency with gradients_impl gradients_impl._function = sys.modules[__name__] # pylint: disable=protected-access +FORWARD_FUNCTION_ATTRIBUTE_NAME = "forward_function_name" +BACKWARD_FUNCTION_ATTRIBUTE_NAME = "backward_function_name" # TODO(scottzhu): Update this to allow arbitrary attribute names in future. -WHITELIST_FUNCTION_ATTRIBUTE_PREFIX = "experimental_" +WHITELIST_FUNCTION_ATTRIBUTE_REGEX = [ + "experimental_.*", + FORWARD_FUNCTION_ATTRIBUTE_NAME, + BACKWARD_FUNCTION_ATTRIBUTE_NAME +] def _create_substitute_placeholder(value, name=None, dtype=None): @@ -140,10 +147,11 @@ def _parse_func_attrs(attributes): """ attrs = {} for key, value in attributes.items(): - if not key.startswith(WHITELIST_FUNCTION_ATTRIBUTE_PREFIX): + if not any([re.match(reg, key) + for reg in WHITELIST_FUNCTION_ATTRIBUTE_REGEX]): raise ValueError("Attribute name is not whitelisted. " "Whitelisted: prefix %s, got: %s" % - (WHITELIST_FUNCTION_ATTRIBUTE_PREFIX, key)) + (WHITELIST_FUNCTION_ATTRIBUTE_REGEX, key)) if isinstance(value, attr_value_pb2.AttrValue): attrs[key] = value @@ -154,7 +162,7 @@ def _parse_func_attrs(attributes): attrs[key] = attr_value_pb2.AttrValue(i=value) elif isinstance(value, float): attrs[key] = attr_value_pb2.AttrValue(f=value) - elif isinstance(value, str): + elif isinstance(value, (str, bytes)): attrs[key] = attr_value_pb2.AttrValue(s=compat.as_bytes(value)) else: raise ValueError("Unsupported attribute type for %s with type %s" % @@ -705,6 +713,7 @@ class Function(object): def _construct_backprop_function(self): """Constructs the backprop function object for this function.""" backwards_graph = FuncGraph(_backward_name(self._func_graph.name)) + forward_function_name = _forward_name(self._func_graph.name) with backwards_graph.as_default(): gradients_wrt_outputs = [ graph_placeholder(x.dtype, x.shape) for x in self._func_graph.outputs @@ -715,11 +724,11 @@ class Function(object): grad_ys=gradients_wrt_outputs, src_graph=self._func_graph) - self._forward_function = _EagerDefinedFunction( - _forward_name( - self._func_graph.name), self._func_graph, self._func_graph.inputs, - self._func_graph.outputs + list(backwards_graph.captures.keys()), - self._attrs) + backwards_graph_captures = list(backwards_graph.captures.keys()) + + backward_function_attr = _parse_func_attrs( + {FORWARD_FUNCTION_ATTRIBUTE_NAME: forward_function_name}) + backward_function_attr.update(self._attrs) # The ordering of `backwards_graph.inputs` is important: inputs of # `self._backward_graph_function` correspond to outputs of @@ -732,7 +741,17 @@ class Function(object): grad for grad in _flatten(gradients_wrt_inputs) if grad is not None) backwards_graph.structured_outputs = gradients_wrt_inputs self._backward_graph_function = Function( - backwards_graph, attrs=self._attrs) + backwards_graph, attrs=backward_function_attr) + + forward_function_attr = _parse_func_attrs({ + BACKWARD_FUNCTION_ATTRIBUTE_NAME: + self._backward_graph_function._inference_function.name}) # pylint: disable=protected-access + forward_function_attr.update(self._attrs) + + self._forward_function = _EagerDefinedFunction( + forward_function_name, self._func_graph, self._func_graph.inputs, + self._func_graph.outputs + backwards_graph_captures, + forward_function_attr) def _backprop_call(self, args): """Calls the forward function and records the result on a tape. diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 34a2648e26..afe3ba9893 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -1687,6 +1687,21 @@ class FunctionTest(test.TestCase): self.assertRegexpMatches(captured_function_names[i], expected_func_name_regex[i]) + # Check the forward and backward function has the correct attributes. + self.assertEquals( + functions[1].definition.attr['backward_function_name'].s, + functions[2].name) + self.assertEquals( + functions[2].definition.attr['forward_function_name'].s, + functions[1].name) + + self.assertEquals( + functions[4].definition.attr['backward_function_name'].s, + functions[5].name) + self.assertEquals( + functions[5].definition.attr['forward_function_name'].s, + functions[4].name) + sq = defun_matmul(t, t) double = add(t, t) self.assertAllEqual(sq.eval().reshape(-1), [7, 10, 15, 22]) -- GitLab From f0c219d095f38f7ce6febfb68d4f84d64aa1829a Mon Sep 17 00:00:00 2001 From: Youlong Cheng Date: Mon, 1 Oct 2018 12:28:32 -0700 Subject: [PATCH 0162/1085] Expose tpu_host_placement_function(). PiperOrigin-RevId: 215259803 --- tensorflow/contrib/tpu/python/tpu/tpu_context.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py index 7cfb6c38fa..da6bdf67d6 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py @@ -154,6 +154,20 @@ class TPUContext(object): # as far as model is replicated to all cores in the system. return self._internal_ctx.device_for_replica(replica_id) + @property + def tpu_host_placement_function(self): + """Returns the TPU host place function. + + The place function takes host_id as the input and returns the TF device + for the correspoding host. + """ + + def _placement_function(host_id): + """Return the host device given host_id.""" + return self._internal_ctx.tpu_host_placement_function(host_id=host_id) + + return _placement_function + class _InternalTPUContext(object): """A context holds immutable states of TPU computation. -- GitLab From 5c8c48df7fd4ccbe4a9dec035fdec6b02a5d6016 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 12:54:56 -0700 Subject: [PATCH 0163/1085] Internal build specification change PiperOrigin-RevId: 215263951 --- tensorflow/core/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 57819cec70..0aae29d10c 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -271,6 +271,12 @@ proto_library( visibility = ["//visibility:public"], ) +java_proto_library( + name = "example_java_proto", + visibility = ["//visibility:public"], + deps = [":example_protos"], +) + closure_proto_library( name = "example_protos_closure", visibility = ["//visibility:public"], -- GitLab From 3648cb0198690d551ea5c8eefcf706c8fa67f4f0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 13:07:12 -0700 Subject: [PATCH 0164/1085] Add option to initialize the TPU system. PiperOrigin-RevId: 215266241 --- tensorflow/python/tools/saved_model_cli.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py index 3dbccd1409..2fcb0fa029 100644 --- a/tensorflow/python/tools/saved_model_cli.py +++ b/tensorflow/python/tools/saved_model_cli.py @@ -267,7 +267,8 @@ def scan_meta_graph_def(meta_graph_def): def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key, input_tensor_key_feed_dict, outdir, - overwrite_flag, worker=None, tf_debug=False): + overwrite_flag, worker=None, init_tpu=False, + tf_debug=False): """Runs SavedModel and fetch all outputs. Runs the input dictionary through the MetaGraphDef within a SavedModel @@ -287,6 +288,8 @@ def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key, the same name exists. worker: If provided, the session will be run on the worker. Valid worker specification is a bns or gRPC path. + init_tpu: If true, the TPU system will be initialized after the session + is created. tf_debug: A boolean flag to use TensorFlow Debugger (TFDBG) to observe the intermediate Tensor values and runtime GraphDefs while running the SavedModel. @@ -328,6 +331,12 @@ def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key, ] with session.Session(worker, graph=ops_lib.Graph()) as sess: + if init_tpu: + print('Initializing TPU System ...') + # This is needed for freshly started worker, or if the job + # restarts after a preemption. + sess.run(tf.contrib.tpu.initialize_system()) + loader.load(sess, tag_set.split(','), saved_model_dir) if tf_debug: @@ -632,7 +641,7 @@ def run(args): run_saved_model_with_feed_dict(args.dir, args.tag_set, args.signature_def, tensor_key_feed_dict, args.outdir, args.overwrite, worker=args.worker, - tf_debug=args.tf_debug) + init_tpu=args.init_tpu, tf_debug=args.tf_debug) def scan(args): @@ -775,6 +784,12 @@ def create_parser(): default=None, help='if specified, a Session will be run on the worker. ' 'Valid worker specification is a bns or gRPC path.') + parser_run.add_argument( + '--init_tpu', + action='store_true', + default=None, + help='if specified, tpu.initialize_system will be called on the Session. ' + 'This option should be only used if the worker is a TPU job.') parser_run.set_defaults(func=run) # scan command -- GitLab From 3c6e6885f32e7638ece306dad3a5081b06137bdc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 1 Oct 2018 13:08:10 -0700 Subject: [PATCH 0165/1085] Check in and refactor the OVIC detector benchmarker. PiperOrigin-RevId: 215266415 --- tensorflow/contrib/lite/java/ovic/BUILD | 61 +++++- .../contrib/lite/java/ovic/demo/app/BUILD | 5 +- .../demo/app/OvicBenchmarkerActivity.java | 77 +++++--- .../demo/app/res/layout/activity_main.xml | 27 ++- .../java/ovic/demo/app/res/values/strings.xml | 3 +- .../java/org/tensorflow/ovic/BoundingBox.java | 68 +++++++ .../org/tensorflow/ovic/OvicBenchmarker.java | 152 ++++++--------- ...ult.java => OvicClassificationResult.java} | 12 +- .../org/tensorflow/ovic/OvicClassifier.java | 10 +- .../ovic/OvicClassifierBenchmarker.java | 142 ++++++++++++++ .../tensorflow/ovic/OvicDetectionResult.java | 91 +++++++++ .../org/tensorflow/ovic/OvicDetector.java | 184 ++++++++++++++++++ .../ovic/OvicDetectorBenchmarker.java | 160 +++++++++++++++ .../org/tensorflow/ovic/OvicValidator.java | 2 +- .../tensorflow/ovic/OvicClassifierTest.java | 6 +- .../org/tensorflow/ovic/OvicDetectorTest.java | 149 ++++++++++++++ .../contrib/lite/java/ovic/src/testdata/BUILD | 5 +- .../java/ovic/src/testdata/coco_labels.txt | 91 +++++++++ 18 files changed, 1101 insertions(+), 144 deletions(-) create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/BoundingBox.java rename tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/{OvicSingleImageResult.java => OvicClassificationResult.java} (83%) create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java create mode 100644 tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicDetectorTest.java create mode 100644 tensorflow/contrib/lite/java/ovic/src/testdata/coco_labels.txt diff --git a/tensorflow/contrib/lite/java/ovic/BUILD b/tensorflow/contrib/lite/java/ovic/BUILD index bb0be04ca2..ea9b9ed4b6 100644 --- a/tensorflow/contrib/lite/java/ovic/BUILD +++ b/tensorflow/contrib/lite/java/ovic/BUILD @@ -9,6 +9,7 @@ licenses(["notice"]) # Apache 2.0 load("//tensorflow/java:build_defs.bzl", "JAVACOPTS") +# Build targets for OVIC classification. java_test( name = "OvicClassifierTest", size = "medium", @@ -45,8 +46,9 @@ android_library( name = "ovicbenchmarkerlib", srcs = [ "src/main/java/org/tensorflow/ovic/OvicBenchmarker.java", + "src/main/java/org/tensorflow/ovic/OvicClassificationResult.java", "src/main/java/org/tensorflow/ovic/OvicClassifier.java", - "src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java", + "src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java", ], manifest = "//tensorflow/contrib/lite/java:AndroidManifest.xml", tags = ["no_oss"], @@ -60,8 +62,8 @@ android_library( java_library( name = "ovicbenchmarkerlib_java", srcs = [ + "src/main/java/org/tensorflow/ovic/OvicClassificationResult.java", "src/main/java/org/tensorflow/ovic/OvicClassifier.java", - "src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java", ], javacopts = JAVACOPTS, tags = ["no_oss"], @@ -73,3 +75,58 @@ java_library( "@org_checkerframework_qual", ], ) + +# Build targets for OVIC detection. +java_test( + name = "OvicDetectorTest", + size = "medium", + srcs = ["src/test/java/org/tensorflow/ovic/OvicDetectorTest.java"], + data = [ + "//tensorflow/contrib/lite/java/ovic/src/testdata:coco_labels.txt", + "//tensorflow/contrib/lite/java/ovic/src/testdata:ovic_testdata", + "@tflite_mobilenet_ssd_quant//:detect.tflite", + ], + javacopts = JAVACOPTS, + tags = ["no_oss"], + test_class = "org.tensorflow.ovic.OvicDetectorTest", + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/contrib/lite/java/ovic:ovicdetectionbenchmarkerlib_java", + "@com_google_truth", + "@junit", + ], +) + +android_library( + name = "ovicdetectionbenchmarkerlib", + srcs = [ + "src/main/java/org/tensorflow/ovic/BoundingBox.java", + "src/main/java/org/tensorflow/ovic/OvicBenchmarker.java", + "src/main/java/org/tensorflow/ovic/OvicDetectionResult.java", + "src/main/java/org/tensorflow/ovic/OvicDetector.java", + "src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java", + ], + manifest = "//tensorflow/contrib/lite/java:AndroidManifest.xml", + deps = [ + "//tensorflow/contrib/lite/java:tensorflowlite", + "//tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper", + "@org_checkerframework_qual", + ], +) + +java_library( + name = "ovicdetectionbenchmarkerlib_java", + srcs = [ + "src/main/java/org/tensorflow/ovic/BoundingBox.java", + "src/main/java/org/tensorflow/ovic/OvicDetectionResult.java", + "src/main/java/org/tensorflow/ovic/OvicDetector.java", + ], + javacopts = JAVACOPTS, + deps = [ + "//tensorflow/contrib/lite/java:libtensorflowlite_jni.so", + "//tensorflow/contrib/lite/java:tensorflowlite_java", + "//tensorflow/contrib/lite/java/src/main/native", + "//tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper", + "@org_checkerframework_qual", + ], +) diff --git a/tensorflow/contrib/lite/java/ovic/demo/app/BUILD b/tensorflow/contrib/lite/java/ovic/demo/app/BUILD index 058240aada..f567358ea3 100644 --- a/tensorflow/contrib/lite/java/ovic/demo/app/BUILD +++ b/tensorflow/contrib/lite/java/ovic/demo/app/BUILD @@ -10,8 +10,10 @@ android_binary( ], aapt_version = "aapt", assets = [ - "//tensorflow/contrib/lite/java/ovic/src/testdata:ovic_testdata", + "//tensorflow/contrib/lite/java/ovic/src/testdata:coco_labels.txt", "//tensorflow/contrib/lite/java/ovic/src/testdata:labels.txt", + "//tensorflow/contrib/lite/java/ovic/src/testdata:ovic_testdata", + "@tflite_mobilenet_ssd_quant//:detect.tflite", ], assets_dir = "", custom_package = "ovic.demo.app", @@ -25,6 +27,7 @@ android_binary( deps = [ "//tensorflow/contrib/lite/java:tensorflowlite", "//tensorflow/contrib/lite/java/ovic:ovicbenchmarkerlib", + "//tensorflow/contrib/lite/java/ovic:ovicdetectionbenchmarkerlib", "@androidsdk//com.android.support:support-v13-25.2.0", "@androidsdk//com.android.support:support-v4-25.2.0", ], diff --git a/tensorflow/contrib/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java b/tensorflow/contrib/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java index 4adf94aeb6..48c29ecebe 100644 --- a/tensorflow/contrib/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java +++ b/tensorflow/contrib/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java @@ -35,19 +35,18 @@ import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.text.DecimalFormat; import org.tensorflow.ovic.OvicBenchmarker; -import org.tensorflow.ovic.OvicSingleImageResult; - +import org.tensorflow.ovic.OvicClassifierBenchmarker; +import org.tensorflow.ovic.OvicDetectorBenchmarker; /** Class that benchmark image classifier models. */ public class OvicBenchmarkerActivity extends Activity { /** Tag for the {@link Log}. */ private static final String TAG = "OvicBenchmarkerActivity"; - /** Name of the label file stored in Assets. */ - private static final String LABEL_PATH = "labels.txt"; - - private static final String TEST_IMAGE_PATH = "test_image_224.jpg"; - private static final String MODEL_PATH = "float_model.lite"; + /** Name of the task-dependent data files stored in Assets. */ + private static String labelPath = null; + private static String testImagePath = null; + private static String modelPath = null; /** * Each bottom press will launch a benchmarking experiment. The experiment stops when either the * total native latency reaches WALL_TIME or the number of iterations reaches MAX_ITERATIONS, @@ -66,8 +65,6 @@ public class OvicBenchmarkerActivity extends Activity { private MappedByteBuffer model = null; private InputStream labelInputStream = null; private OvicBenchmarker benchmarker; - /** Inference result of each iteration. */ - OvicSingleImageResult iterResult = null; private TextView textView = null; // private Button startButton = null; @@ -83,21 +80,31 @@ public class OvicBenchmarkerActivity extends Activity { } private Bitmap loadTestBitmap() throws IOException { - InputStream imageStream = getAssets().open(TEST_IMAGE_PATH); + InputStream imageStream = getAssets().open(testImagePath); return BitmapFactory.decodeStream(imageStream); } - public void initializeTest() throws IOException { + public void initializeTest(boolean benchmarkClassification) throws IOException { Log.i(TAG, "Initializing benchmarker."); - benchmarker = new OvicBenchmarker(WALL_TIME); + if (benchmarkClassification) { + benchmarker = new OvicClassifierBenchmarker(WALL_TIME); + labelPath = "labels.txt"; + testImagePath = "test_image_224.jpg"; + modelPath = "quantized_model.lite"; + } else { // Benchmarking detection. + benchmarker = new OvicDetectorBenchmarker(WALL_TIME); + labelPath = "coco_labels.txt"; + testImagePath = "test_image_224.jpg"; + modelPath = "detect.tflite"; + } AssetManager am = getAssets(); - AssetFileDescriptor fileDescriptor = am.openFd(MODEL_PATH); + AssetFileDescriptor fileDescriptor = am.openFd(modelPath); FileInputStream modelInputStream = new FileInputStream(fileDescriptor.getFileDescriptor()); FileChannel fileChannel = modelInputStream.getChannel(); long startOffset = fileDescriptor.getStartOffset(); long declaredLength = fileDescriptor.getDeclaredLength(); model = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); - labelInputStream = am.open(LABEL_PATH); + labelInputStream = am.open(labelPath); } public Boolean doTestIteration() throws IOException, InterruptedException { @@ -117,24 +124,44 @@ public class OvicBenchmarkerActivity extends Activity { Log.i(TAG, "Going to do test iter."); // Start testing. Bitmap testImageBitmap = loadTestBitmap(); - iterResult = benchmarker.doTestIteration(testImageBitmap); - testImageBitmap.recycle(); - if (iterResult == null) { + try { + if (!benchmarker.processBitmap(testImageBitmap)) { + throw new RuntimeException("Failed to run test."); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + testImageBitmap.recycle(); + } + String iterResultString = benchmarker.getLastResultString(); + if (iterResultString == null) { throw new RuntimeException("Inference failed to produce a result."); } - Log.i(TAG, iterResult.toString()); + Log.i(TAG, iterResultString); return true; } - public void startPressed(View view) throws IOException { - Log.i(TAG, "Start pressed"); + public void detectPressed(View view) throws IOException { + benchmarkSession(false); + } + public void classifyPressed(View view) throws IOException { + benchmarkSession(true); + } + + private void benchmarkSession(boolean benchmarkClassification) throws IOException { try { - initializeTest(); + initializeTest(benchmarkClassification); } catch (IOException e) { Log.e(TAG, "Can't initialize benchmarker.", e); throw e; } String displayText = ""; + if (benchmarkClassification) { + displayText = "Classification benchmark: "; + } else { + displayText = "Detection benchmark: "; + } try { setProcessorAffinity(BIG_CORE_MASK); } catch (IOException e) { @@ -144,7 +171,6 @@ public class OvicBenchmarkerActivity extends Activity { Log.i(TAG, "Successfully initialized benchmarker."); int testIter = 0; Boolean iterSuccess = false; - double totalLatency = 0.0f; while (testIter < MAX_ITERATIONS) { try { iterSuccess = doTestIteration(); @@ -153,23 +179,22 @@ public class OvicBenchmarkerActivity extends Activity { throw e; } catch (InterruptedException e) { Log.e(TAG, "Interrupted at iteration " + testIter); + displayText += e.getMessage() + "\n"; } if (!iterSuccess) { break; } testIter++; - totalLatency += (double) iterResult.latency; } - ; Log.i(TAG, "Benchmarking finished"); if (textView != null) { if (testIter > 0) { textView.setText( displayText - + MODEL_PATH + + modelPath + ": Average latency=" - + df2.format(totalLatency / testIter) + + df2.format(benchmarker.getTotalRunTime() / testIter) + "ms after " + testIter + " runs."); diff --git a/tensorflow/contrib/lite/java/ovic/demo/app/res/layout/activity_main.xml b/tensorflow/contrib/lite/java/ovic/demo/app/res/layout/activity_main.xml index e9d83bae54..1bce60ff7d 100644 --- a/tensorflow/contrib/lite/java/ovic/demo/app/res/layout/activity_main.xml +++ b/tensorflow/contrib/lite/java/ovic/demo/app/res/layout/activity_main.xml @@ -30,14 +30,14 @@ android:layout_height="wrap_content" android:text="@string/initial_status_msg" android:id="@+id/textView" - android:layout_above="@+id/button_start" + android:layout_above="@+id/button_clf_start" android:layout_alignParentTop="true"/>